From 27eb3ca6eece3eb24104b9c397b7ade5a359e0d6 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Tue, 10 Mar 2020 16:10:53 -0400 Subject: [PATCH 01/42] fixed bugs added for the bug bounty --- bugs.txt | 9 ++ snow/consensus/avalanche/topological.go | 19 +++ snow/consensus/avalanche/topological_test.go | 72 +++++++++++ snow/consensus/snowball/unary_snowball.go | 2 + .../consensus/snowball/unary_snowball_test.go | 30 ++++- snow/engine/avalanche/bootstrapper.go | 6 + snow/engine/avalanche/bootstrapper_test.go | 108 ++++++++++++++++ snow/engine/avalanche/issuer.go | 7 +- snow/engine/avalanche/polls.go | 22 ++-- snow/engine/avalanche/transitive_test.go | 117 ++++++++++++++++++ snow/engine/snowman/bootstrapper.go | 6 + snow/engine/snowman/bootstrapper_test.go | 110 ++++++++++++++++ snow/engine/snowman/polls.go | 33 +++-- snow/engine/snowman/transitive.go | 14 ++- snow/engine/snowman/transitive_test.go | 112 +++++++++++++++++ 15 files changed, 630 insertions(+), 37 deletions(-) create mode 100644 bugs.txt diff --git a/bugs.txt b/bugs.txt new file mode 100644 index 0000000..984fcef --- /dev/null +++ b/bugs.txt @@ -0,0 +1,9 @@ +Added bugs: + +- Inside of gecko/snow/consensus/avalanche/topological.go#Topological.pushVotes the votes must be properly filtered. Specifically, a byzantine node should not be able to vote for two different transactions that conflict with each other during the same poll. If a node votes for conflicting transactions during the same poll, either all the node's votes should be dropped, or the votes for the conflicting transactions should be dropped. + +- Inside of gecko/snow/consensus/snowball/unary_snowball.go#unarySnowball.Extend the confidence and bias fields should have been set to the values in the unary snowball instance. + +- Inside of gecko/snow/engine/avalanche/bootstrapper.go#bootstrapper.Put and gecko/snow/engine/snowman/bootstrapper.go#bootstrapper.Put the engine must check that the provided vtx/blk ID is the ID of the parsed container. Otherwise, a byzantine node could send a container and report the wrong container ID for it. This would allow an un-intended container to be marked as accepted during bootstrapping. + +- Inside of gecko/snow/engine/avalanche/polls.go#poll and gecko/snow/engine/snowman/polls.go#poll the poll should only allow a validator to vote once per poll. Also, this validator must have been part of set of validators that was polled during the query. \ No newline at end of file diff --git a/snow/consensus/avalanche/topological.go b/snow/consensus/avalanche/topological.go index dca8a19..5e6212c 100644 --- a/snow/consensus/avalanche/topological.go +++ b/snow/consensus/avalanche/topological.go @@ -257,6 +257,7 @@ func (ta *Topological) pushVotes( kahnNodes map[[32]byte]kahnNode, leaves []ids.ID) ids.Bag { votes := make(ids.UniqueBag) + txConflicts := make(map[[32]byte]ids.Set) for len(leaves) > 0 { newLeavesSize := len(leaves) - 1 @@ -271,6 +272,12 @@ func (ta *Topological) pushVotes( // Give the votes to the consumer txID := tx.ID() votes.UnionSet(txID, kahn.votes) + + // Map txID to set of Conflicts + txKey := txID.Key() + if _, exists := txConflicts[txKey]; !exists { + txConflicts[txKey] = ta.cg.Conflicts(tx) + } } for _, dep := range vtx.Parents() { @@ -291,6 +298,18 @@ func (ta *Topological) pushVotes( } } + // Create bag of votes for conflicting transactions + conflictingVotes := make(ids.UniqueBag) + for txHash, conflicts := range txConflicts { + txID := ids.NewID(txHash) + for conflictTxHash := range conflicts { + conflictTxID := ids.NewID(conflictTxHash) + conflictingVotes.UnionSet(txID, votes.GetSet(conflictTxID)) + } + } + + votes.Difference(&conflictingVotes) + return votes.Bag(ta.params.Alpha) } diff --git a/snow/consensus/avalanche/topological_test.go b/snow/consensus/avalanche/topological_test.go index f43ee5b..c28d567 100644 --- a/snow/consensus/avalanche/topological_test.go +++ b/snow/consensus/avalanche/topological_test.go @@ -103,6 +103,78 @@ func TestAvalancheVoting(t *testing.T) { } } +func TestAvalancheIgnoreInvalidVoting(t *testing.T) { + params := Parameters{ + Parameters: snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 3, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 1, + }, + Parents: 2, + BatchSize: 1, + } + + vts := []Vertex{&Vtx{ + id: GenerateID(), + status: choices.Accepted, + }, &Vtx{ + id: GenerateID(), + status: choices.Accepted, + }} + utxos := []ids.ID{GenerateID()} + + ta := Topological{} + ta.Initialize(snow.DefaultContextTest(), params, vts) + + tx0 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx0.Ins.Add(utxos[0]) + + vtx0 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx0}, + height: 1, + status: choices.Processing, + } + + tx1 := &snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + } + tx1.Ins.Add(utxos[0]) + + vtx1 := &Vtx{ + dependencies: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx1}, + height: 1, + status: choices.Processing, + } + + ta.Add(vtx0) + ta.Add(vtx1) + + sm := make(ids.UniqueBag) + + sm.Add(0, vtx0.id) + sm.Add(1, vtx1.id) + + // Add Illegal Vote cast by Response 2 + sm.Add(2, vtx0.id) + sm.Add(2, vtx1.id) + + ta.RecordPoll(sm) + + if ta.Finalized() { + t.Fatalf("An avalanche instance finalized too early") + } +} + func TestAvalancheTransitiveVoting(t *testing.T) { params := Parameters{ Parameters: snowball.Parameters{ diff --git a/snow/consensus/snowball/unary_snowball.go b/snow/consensus/snowball/unary_snowball.go index 6d0db07..3999a74 100644 --- a/snow/consensus/snowball/unary_snowball.go +++ b/snow/consensus/snowball/unary_snowball.go @@ -48,9 +48,11 @@ func (sb *unarySnowball) Extend(beta int, choice int) BinarySnowball { snowflake: binarySnowflake{ beta: beta, preference: choice, + confidence: sb.confidence, finalized: sb.Finalized(), }, } + bs.numSuccessfulPolls[choice] = sb.numSuccessfulPolls return bs } diff --git a/snow/consensus/snowball/unary_snowball_test.go b/snow/consensus/snowball/unary_snowball_test.go index 8bf098a..224cd4c 100644 --- a/snow/consensus/snowball/unary_snowball_test.go +++ b/snow/consensus/snowball/unary_snowball_test.go @@ -42,11 +42,32 @@ func TestUnarySnowball(t *testing.T) { binarySnowball := sbClone.Extend(beta, 0) + expected := "SB(Preference = 0, NumSuccessfulPolls[0] = 2, NumSuccessfulPolls[1] = 0, SF = SF(Preference = 0, Confidence = 1, Finalized = false))" + if result := binarySnowball.String(); result != expected { + t.Fatalf("Expected:\n%s\nReturned:\n%s", expected, result) + } + binarySnowball.RecordUnsuccessfulPoll() + for i := 0; i < 3; i++ { + if binarySnowball.Preference() != 0 { + t.Fatalf("Wrong preference") + } else if binarySnowball.Finalized() { + t.Fatalf("Should not have finalized") + } + binarySnowball.RecordSuccessfulPoll(1) + binarySnowball.RecordUnsuccessfulPoll() + } + + if binarySnowball.Preference() != 1 { + t.Fatalf("Wrong preference") + } else if binarySnowball.Finalized() { + t.Fatalf("Should not have finalized") + } binarySnowball.RecordSuccessfulPoll(1) - - if binarySnowball.Finalized() { + if binarySnowball.Preference() != 1 { + t.Fatalf("Wrong preference") + } else if binarySnowball.Finalized() { t.Fatalf("Should not have finalized") } @@ -57,4 +78,9 @@ func TestUnarySnowball(t *testing.T) { } else if !binarySnowball.Finalized() { t.Fatalf("Should have finalized") } + + expected = "SB(NumSuccessfulPolls = 2, Confidence = 1, Finalized = false)" + if str := sb.String(); str != expected { + t.Fatalf("Wrong state. Expected:\n%s\nGot:\n%s", expected, str) + } } diff --git a/snow/engine/avalanche/bootstrapper.go b/snow/engine/avalanche/bootstrapper.go index 7d3d7c8..26b202e 100644 --- a/snow/engine/avalanche/bootstrapper.go +++ b/snow/engine/avalanche/bootstrapper.go @@ -103,6 +103,12 @@ func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxB return } + if realVtxID := vtx.ID(); !vtxID.Equals(realVtxID) { + b.BootstrapConfig.Context.Log.Warn("Put called for vertexID %s, but provided vertexID %s", vtxID, realVtxID) + b.GetFailed(vdr, requestID, vtxID) + return + } + b.addVertex(vtx) } diff --git a/snow/engine/avalanche/bootstrapper_test.go b/snow/engine/avalanche/bootstrapper_test.go index d1be936..6f581b5 100644 --- a/snow/engine/avalanche/bootstrapper_test.go +++ b/snow/engine/avalanche/bootstrapper_test.go @@ -334,6 +334,114 @@ func TestBootstrapperUnknownByzantineResponse(t *testing.T) { } } +func TestBootstrapperWrongIDByzantineResponse(t *testing.T) { + config, peerID, sender, state, _ := newConfig(t) + + vtxID0 := ids.Empty.Prefix(0) + vtxID1 := ids.Empty.Prefix(1) + + vtxBytes0 := []byte{0} + vtxBytes1 := []byte{1} + + vtx0 := &Vtx{ + id: vtxID0, + height: 0, + status: choices.Processing, + bytes: vtxBytes0, + } + vtx1 := &Vtx{ + id: vtxID1, + height: 0, + status: choices.Processing, + bytes: vtxBytes1, + } + + bs := bootstrapper{} + bs.metrics.Initialize(config.Context.Log, fmt.Sprintf("gecko_%s", config.Context.ChainID), prometheus.NewRegistry()) + bs.Initialize(config) + + acceptedIDs := ids.Set{} + acceptedIDs.Add( + vtxID0, + ) + + state.getVertex = func(vtxID ids.ID) (avalanche.Vertex, error) { + switch { + case vtxID.Equals(vtxID0): + return nil, errUnknownVertex + default: + t.Fatal(errUnknownVertex) + panic(errUnknownVertex) + } + } + + requestID := new(uint32) + sender.GetF = func(vdr ids.ShortID, reqID uint32, vtxID ids.ID) { + if !vdr.Equals(peerID) { + t.Fatalf("Should have requested vertex from %s, requested from %s", peerID, vdr) + } + switch { + case vtxID.Equals(vtxID0): + default: + t.Fatalf("Requested unknown vertex") + } + + *requestID = reqID + } + + bs.ForceAccepted(acceptedIDs) + + state.getVertex = nil + sender.GetF = nil + + state.parseVertex = func(vtxBytes []byte) (avalanche.Vertex, error) { + switch { + case bytes.Equal(vtxBytes, vtxBytes0): + return vtx0, nil + case bytes.Equal(vtxBytes, vtxBytes1): + return vtx1, nil + } + t.Fatal(errParsedUnknownVertex) + return nil, errParsedUnknownVertex + } + + state.getVertex = func(vtxID ids.ID) (avalanche.Vertex, error) { + switch { + case vtxID.Equals(vtxID0): + return vtx0, nil + case vtxID.Equals(vtxID1): + return vtx1, nil + default: + t.Fatal(errUnknownVertex) + panic(errUnknownVertex) + } + } + + finished := new(bool) + bs.onFinished = func() { *finished = true } + sender.CantGet = false + + bs.Put(peerID, *requestID, vtxID0, vtxBytes1) + + sender.CantGet = true + + bs.Put(peerID, *requestID, vtxID0, vtxBytes0) + + state.parseVertex = nil + state.edge = nil + bs.onFinished = nil + + if !*finished { + t.Fatalf("Bootstrapping should have finished") + } + if vtx0.Status() != choices.Accepted { + t.Fatalf("Vertex should be accepted") + } + if vtx1.Status() != choices.Processing { + t.Fatalf("Vertex should be processing") + } +} + func TestBootstrapperVertexDependencies(t *testing.T) { config, peerID, sender, state, _ := newConfig(t) diff --git a/snow/engine/avalanche/issuer.go b/snow/engine/avalanche/issuer.go index befe973..f953fe6 100644 --- a/snow/engine/avalanche/issuer.go +++ b/snow/engine/avalanche/issuer.go @@ -64,9 +64,12 @@ func (i *issuer) Update() { vdrSet.Add(vdr.ID()) } + toSample := ids.ShortSet{} // Copy to a new variable because we may remove an element in sender.Sender + toSample.Union(vdrSet) // and we don't want that to affect the set of validators we wait for [ie vdrSet] + i.t.RequestID++ - if numVdrs := len(vdrs); numVdrs == p.K && i.t.polls.Add(i.t.RequestID, vdrSet.Len()) { - i.t.Config.Sender.PushQuery(vdrSet, i.t.RequestID, vtxID, i.vtx.Bytes()) + if numVdrs := len(vdrs); numVdrs == p.K && i.t.polls.Add(i.t.RequestID, vdrSet) { + i.t.Config.Sender.PushQuery(toSample, i.t.RequestID, vtxID, i.vtx.Bytes()) } else if numVdrs < p.K { i.t.Config.Context.Log.Error("Query for %s was dropped due to an insufficient number of validators", vtxID) } diff --git a/snow/engine/avalanche/polls.go b/snow/engine/avalanche/polls.go index 282fe6a..fa1e7df 100644 --- a/snow/engine/avalanche/polls.go +++ b/snow/engine/avalanche/polls.go @@ -38,10 +38,10 @@ type polls struct { // Add to the current set of polls // Returns true if the poll was registered correctly and the network sample // should be made. -func (p *polls) Add(requestID uint32, numPolled int) bool { +func (p *polls) Add(requestID uint32, vdrs ids.ShortSet) bool { poll, exists := p.m[requestID] if !exists { - poll.numPending = numPolled + poll.polled = vdrs p.m[requestID] = poll p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics @@ -59,7 +59,7 @@ func (p *polls) Vote(requestID uint32, vdr ids.ShortID, votes []ids.ID) (ids.Uni return nil, false } - poll.Vote(votes) + poll.Vote(votes, vdr) if poll.Finished() { p.log.Verbo("Poll is finished") delete(p.m, requestID) @@ -83,19 +83,19 @@ func (p *polls) String() string { // poll represents the current state of a network poll for a vertex type poll struct { - votes ids.UniqueBag - numPending int + votes ids.UniqueBag + polled ids.ShortSet } // Vote registers a vote for this poll -func (p *poll) Vote(votes []ids.ID) { - if p.numPending > 0 { - p.numPending-- - p.votes.Add(uint(p.numPending), votes...) +func (p *poll) Vote(votes []ids.ID, vdr ids.ShortID) { + if p.polled.Contains(vdr) { + p.polled.Remove(vdr) + p.votes.Add(uint(p.polled.Len()), votes...) } } // Finished returns true if the poll has completed, with no more required // responses -func (p poll) Finished() bool { return p.numPending <= 0 } -func (p poll) String() string { return fmt.Sprintf("Waiting on %d chits", p.numPending) } +func (p poll) Finished() bool { return p.polled.Len() == 0 } +func (p poll) String() string { return fmt.Sprintf("Waiting on %d chits", p.polled.Len()) } diff --git a/snow/engine/avalanche/transitive_test.go b/snow/engine/avalanche/transitive_test.go index 6f5b5ed..15b258b 100644 --- a/snow/engine/avalanche/transitive_test.go +++ b/snow/engine/avalanche/transitive_test.go @@ -2363,3 +2363,120 @@ func TestEngineBootstrappingIntoConsensus(t *testing.T) { sender.PushQueryF = nil st.getVertex = nil } + +func TestEngineDoubleChit(t *testing.T) { + config := DefaultConfig() + + config.Params.Alpha = 2 + config.Params.K = 2 + + vdr0 := validators.GenerateRandomValidator(1) + vdr1 := validators.GenerateRandomValidator(1) + vals := validators.NewSet() + vals.Add(vdr0) + vals.Add(vdr1) + config.Validators = vals + + sender := &common.SenderTest{} + sender.T = t + config.Sender = sender + + sender.Default(true) + sender.CantGetAcceptedFrontier = false + + st := &stateTest{t: t} + config.State = st + + st.Default(true) + + gVtx := &Vtx{ + id: GenerateID(), + status: choices.Accepted, + } + mVtx := &Vtx{ + id: GenerateID(), + status: choices.Accepted, + } + + vts := []avalanche.Vertex{gVtx, mVtx} + utxos := []ids.ID{GenerateID()} + + tx := &TestTx{ + TestTx: snowstorm.TestTx{ + Identifier: GenerateID(), + Stat: choices.Processing, + }, + } + tx.Ins.Add(utxos[0]) + + vtx := &Vtx{ + parents: vts, + id: GenerateID(), + txs: []snowstorm.Tx{tx}, + height: 1, + status: choices.Processing, + bytes: []byte{1, 1, 2, 3}, + } + + st.edge = func() []ids.ID { return []ids.ID{vts[0].ID(), vts[1].ID()} } + st.getVertex = func(id ids.ID) (avalanche.Vertex, error) { + switch { + case id.Equals(gVtx.ID()): + return gVtx, nil + case id.Equals(mVtx.ID()): + return mVtx, nil + } + t.Fatalf("Unknown vertex") + panic("Should have errored") + } + + te := &Transitive{} + te.Initialize(config) + te.finishBootstrapping() + + reqID := new(uint32) + sender.PushQueryF = func(inVdrs ids.ShortSet, requestID uint32, vtxID ids.ID, _ []byte) { + *reqID = requestID + if inVdrs.Len() != 2 { + t.Fatalf("Wrong number of validators") + } + if !vtxID.Equals(vtx.ID()) { + t.Fatalf("Wrong vertex requested") + } + } + st.getVertex = func(id ids.ID) (avalanche.Vertex, error) { + switch { + case id.Equals(vtx.ID()): + return vtx, nil + } + t.Fatalf("Unknown vertex") + panic("Should have errored") + } + + te.insert(vtx) + + votes := ids.Set{} + votes.Add(vtx.ID()) + + if status := tx.Status(); status != choices.Processing { + t.Fatalf("Wrong tx status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr0.ID(), *reqID, votes) + + if status := tx.Status(); status != choices.Processing { + t.Fatalf("Wrong tx status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr0.ID(), *reqID, votes) + + if status := tx.Status(); status != choices.Processing { + t.Fatalf("Wrong tx status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr1.ID(), *reqID, votes) + + if status := tx.Status(); status != choices.Accepted { + t.Fatalf("Wrong tx status: %s ; expected: %s", status, choices.Accepted) + } +} diff --git a/snow/engine/snowman/bootstrapper.go b/snow/engine/snowman/bootstrapper.go index 88724ed..2c0415e 100644 --- a/snow/engine/snowman/bootstrapper.go +++ b/snow/engine/snowman/bootstrapper.go @@ -97,6 +97,12 @@ func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, blkID ids.ID, blkB return } + if realBlkID := blk.ID(); !blkID.Equals(realBlkID) { + b.BootstrapConfig.Context.Log.Warn("Put called for blockID %s, but provided blockID %s", blkID, realBlkID) + b.GetFailed(vdr, requestID, blkID) + return + } + b.addBlock(blk) } diff --git a/snow/engine/snowman/bootstrapper_test.go b/snow/engine/snowman/bootstrapper_test.go index 9cb0968..a972a23 100644 --- a/snow/engine/snowman/bootstrapper_test.go +++ b/snow/engine/snowman/bootstrapper_test.go @@ -252,6 +252,116 @@ func TestBootstrapperUnknownByzantineResponse(t *testing.T) { } } +func TestBootstrapperWrongIDByzantineResponse(t *testing.T) { + config, peerID, sender, vm := newConfig(t) + + blkID0 := ids.Empty.Prefix(0) + blkID1 := ids.Empty.Prefix(1) + blkID2 := ids.Empty.Prefix(2) + + blkBytes0 := []byte{0} + blkBytes1 := []byte{1} + blkBytes2 := []byte{2} + + blk0 := &Blk{ + id: blkID0, + height: 0, + status: choices.Accepted, + bytes: blkBytes0, + } + blk1 := &Blk{ + parent: blk0, + id: blkID1, + height: 1, + status: choices.Processing, + bytes: blkBytes1, + } + blk2 := &Blk{ + parent: blk1, + id: blkID2, + height: 2, + status: choices.Processing, + bytes: blkBytes2, + } + + bs := bootstrapper{} + bs.metrics.Initialize(config.Context.Log, fmt.Sprintf("gecko_%s", config.Context.ChainID), prometheus.NewRegistry()) + bs.Initialize(config) + + acceptedIDs := ids.Set{} + acceptedIDs.Add(blkID1) + + vm.GetBlockF = func(blkID ids.ID) (snowman.Block, error) { + switch { + case blkID.Equals(blkID1): + return nil, errUnknownBlock + default: + t.Fatal(errUnknownBlock) + panic(errUnknownBlock) + } + } + + requestID := new(uint32) + sender.GetF = func(vdr ids.ShortID, reqID uint32, vtxID ids.ID) { + if !vdr.Equals(peerID) { + t.Fatalf("Should have requested block from %s, requested from %s", peerID, vdr) + } + switch { + case vtxID.Equals(blkID1): + default: + t.Fatalf("Requested unknown block") + } + + *requestID = reqID + } + + bs.ForceAccepted(acceptedIDs) + + vm.GetBlockF = nil + sender.GetF = nil + + vm.ParseBlockF = func(blkBytes []byte) (snowman.Block, error) { + switch { + case bytes.Equal(blkBytes, blkBytes2): + return blk2, nil + } + t.Fatal(errUnknownBlock) + return nil, errUnknownBlock + } + + sender.CantGet = false + + bs.Put(peerID, *requestID, blkID1, blkBytes2) + + sender.CantGet = true + + vm.ParseBlockF = func(blkBytes []byte) (snowman.Block, error) { + switch { + case bytes.Equal(blkBytes, blkBytes1): + return blk1, nil + } + t.Fatal(errUnknownBlock) + return nil, errUnknownBlock + } + + finished := new(bool) + bs.onFinished = func() { *finished = true } + + bs.Put(peerID, *requestID, blkID1, blkBytes1) + + vm.ParseBlockF = nil + + if !*finished { + t.Fatalf("Bootstrapping should have finished") + } + if blk1.Status() != choices.Accepted { + t.Fatalf("Block should be accepted") + } + if blk2.Status() != choices.Processing { + t.Fatalf("Block should be processing") + } +} + func TestBootstrapperDependency(t *testing.T) { config, peerID, sender, vm := newConfig(t) diff --git a/snow/engine/snowman/polls.go b/snow/engine/snowman/polls.go index 6e666dc..6765ff7 100644 --- a/snow/engine/snowman/polls.go +++ b/snow/engine/snowman/polls.go @@ -22,11 +22,11 @@ type polls struct { // Add to the current set of polls // Returns true if the poll was registered correctly and the network sample // should be made. -func (p *polls) Add(requestID uint32, numPolled int) bool { +func (p *polls) Add(requestID uint32, vdrs ids.ShortSet) bool { poll, exists := p.m[requestID] if !exists { poll.alpha = p.alpha - poll.numPolled = numPolled + poll.polled = vdrs p.m[requestID] = poll p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics @@ -42,7 +42,7 @@ func (p *polls) Vote(requestID uint32, vdr ids.ShortID, vote ids.ID) (ids.Bag, b if !exists { return ids.Bag{}, false } - poll.Vote(vote) + poll.Vote(vote, vdr) if poll.Finished() { delete(p.m, requestID) p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics @@ -60,7 +60,7 @@ func (p *polls) CancelVote(requestID uint32, vdr ids.ShortID) (ids.Bag, bool) { return ids.Bag{}, false } - poll.CancelVote() + poll.CancelVote(vdr) if poll.Finished() { delete(p.m, requestID) p.numPolls.Set(float64(len(p.m))) // Tracks performance statistics @@ -83,22 +83,18 @@ func (p *polls) String() string { // poll represents the current state of a network poll for a block type poll struct { - alpha int - votes ids.Bag - numPolled int + alpha int + votes ids.Bag + polled ids.ShortSet } // Vote registers a vote for this poll -func (p *poll) CancelVote() { - if p.numPolled > 0 { - p.numPolled-- - } -} +func (p *poll) CancelVote(vdr ids.ShortID) { p.polled.Remove(vdr) } // Vote registers a vote for this poll -func (p *poll) Vote(vote ids.ID) { - if p.numPolled > 0 { - p.numPolled-- +func (p *poll) Vote(vote ids.ID, vdr ids.ShortID) { + if p.polled.Contains(vdr) { + p.polled.Remove(vdr) p.votes.Add(vote) } } @@ -106,13 +102,14 @@ func (p *poll) Vote(vote ids.ID) { // Finished returns true if the poll has completed, with no more required // responses func (p poll) Finished() bool { + remaining := p.polled.Len() received := p.votes.Len() _, freq := p.votes.Mode() - return p.numPolled == 0 || // All k nodes responded + return remaining == 0 || // All k nodes responded freq >= p.alpha || // An alpha majority has returned - received+p.numPolled < p.alpha // An alpha majority can never return + received+remaining < p.alpha // An alpha majority can never return } func (p poll) String() string { - return fmt.Sprintf("Waiting on %d chits", p.numPolled) + return fmt.Sprintf("Waiting on %d chits from %s", p.polled.Len(), p.polled) } diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go index e023a7d..9e97f0a 100644 --- a/snow/engine/snowman/transitive.go +++ b/snow/engine/snowman/transitive.go @@ -297,9 +297,12 @@ func (t *Transitive) pullSample(blkID ids.ID) { vdrSet.Add(vdr.ID()) } + toSample := ids.ShortSet{} + toSample.Union(vdrSet) + t.RequestID++ - if numVdrs := len(vdrs); numVdrs == p.K && t.polls.Add(t.RequestID, vdrSet.Len()) { - t.Config.Sender.PullQuery(vdrSet, t.RequestID, blkID) + if numVdrs := len(vdrs); numVdrs == p.K && t.polls.Add(t.RequestID, vdrSet) { + t.Config.Sender.PullQuery(toSample, t.RequestID, blkID) } else if numVdrs < p.K { t.Config.Context.Log.Error("Query for %s was dropped due to an insufficient number of validators", blkID) } @@ -314,9 +317,12 @@ func (t *Transitive) pushSample(blk snowman.Block) { vdrSet.Add(vdr.ID()) } + toSample := ids.ShortSet{} + toSample.Union(vdrSet) + t.RequestID++ - if numVdrs := len(vdrs); numVdrs == p.K && t.polls.Add(t.RequestID, vdrSet.Len()) { - t.Config.Sender.PushQuery(vdrSet, t.RequestID, blk.ID(), blk.Bytes()) + if numVdrs := len(vdrs); numVdrs == p.K && t.polls.Add(t.RequestID, vdrSet) { + t.Config.Sender.PushQuery(toSample, t.RequestID, blk.ID(), blk.Bytes()) } else if numVdrs < p.K { t.Config.Context.Log.Error("Query for %s was dropped due to an insufficient number of validators", blk.ID()) } diff --git a/snow/engine/snowman/transitive_test.go b/snow/engine/snowman/transitive_test.go index 1920d8c..c97f2f0 100644 --- a/snow/engine/snowman/transitive_test.go +++ b/snow/engine/snowman/transitive_test.go @@ -1076,3 +1076,115 @@ func TestEngineRetryFetch(t *testing.T) { t.Fatalf("Should have requested the block again") } } + +func TestEngineDoubleChit(t *testing.T) { + config := DefaultConfig() + + config.Params = snowball.Parameters{ + Metrics: prometheus.NewRegistry(), + K: 2, + Alpha: 2, + BetaVirtuous: 1, + BetaRogue: 2, + } + + vdr0 := validators.GenerateRandomValidator(1) + vdr1 := validators.GenerateRandomValidator(1) + + vals := validators.NewSet() + config.Validators = vals + + vals.Add(vdr0) + vals.Add(vdr1) + + sender := &common.SenderTest{} + sender.T = t + config.Sender = sender + + sender.Default(true) + + vm := &VMTest{} + vm.T = t + config.VM = vm + + vm.Default(true) + vm.CantSetPreference = false + + gBlk := &Blk{ + id: GenerateID(), + status: choices.Accepted, + } + + vm.LastAcceptedF = func() ids.ID { return gBlk.ID() } + sender.CantGetAcceptedFrontier = false + + te := &Transitive{} + te.Initialize(config) + te.finishBootstrapping() + + vm.LastAcceptedF = nil + sender.CantGetAcceptedFrontier = true + + blk := &Blk{ + parent: gBlk, + id: GenerateID(), + status: choices.Processing, + bytes: []byte{1}, + } + + queried := new(bool) + queryRequestID := new(uint32) + sender.PushQueryF = func(inVdrs ids.ShortSet, requestID uint32, blkID ids.ID, blkBytes []byte) { + if *queried { + t.Fatalf("Asked multiple times") + } + *queried = true + *queryRequestID = requestID + vdrSet := ids.ShortSet{} + vdrSet.Add(vdr0.ID(), vdr1.ID()) + if !inVdrs.Equals(vdrSet) { + t.Fatalf("Asking wrong validator for preference") + } + if !blk.ID().Equals(blkID) { + t.Fatalf("Asking for wrong block") + } + } + + te.insert(blk) + + vm.GetBlockF = func(id ids.ID) (snowman.Block, error) { + switch { + case id.Equals(gBlk.ID()): + return gBlk, nil + case id.Equals(blk.ID()): + return blk, nil + } + t.Fatalf("Unknown block") + panic("Should have errored") + } + + blkSet := ids.Set{} + blkSet.Add(blk.ID()) + + if status := blk.Status(); status != choices.Processing { + t.Fatalf("Wrong status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr0.ID(), *queryRequestID, blkSet) + + if status := blk.Status(); status != choices.Processing { + t.Fatalf("Wrong status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr0.ID(), *queryRequestID, blkSet) + + if status := blk.Status(); status != choices.Processing { + t.Fatalf("Wrong status: %s ; expected: %s", status, choices.Processing) + } + + te.Chits(vdr1.ID(), *queryRequestID, blkSet) + + if status := blk.Status(); status != choices.Accepted { + t.Fatalf("Wrong status: %s ; expected: %s", status, choices.Accepted) + } +} From 8e8dd7529b730f346f2cf0cf8e5659911a962017 Mon Sep 17 00:00:00 2001 From: Shashank Date: Tue, 2 Jun 2020 22:47:02 +0530 Subject: [PATCH 02/42] Fix for KeyStore DoS vulnerability https://github.com/ava-labs/gecko/issues/195 --- api/keystore/service.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 5575083..c1e1c56 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -148,9 +148,17 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre return fmt.Errorf("user already exists: %s", args.Username) } - if zxcvbn.PasswordStrength(args.Password, nil).Score < requiredPassScore { - return errWeakPassword - } + if len(args.Password) < 50 { + if zxcvbn.PasswordStrength(args.Password, nil).Score < requiredPassScore { + return errWeakPassword + } + } + + if len(args.Password) >= 50 { + if zxcvbn.PasswordStrength(args.Password[:50], nil).Score < requiredPassScore { + return errWeakPassword + } + } usr := &User{} if err := usr.Initialize(args.Password); err != nil { From 6dc67bbf7024535ee130388a81e34302d93411ec Mon Sep 17 00:00:00 2001 From: Shashank Date: Wed, 3 Jun 2020 14:08:57 +0530 Subject: [PATCH 03/42] Updated fix for issue 195 https://github.com/ava-labs/gecko/issues/195 --- api/keystore/service.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index c1e1c56..72c7c24 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -148,17 +148,17 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre return fmt.Errorf("user already exists: %s", args.Username) } - if len(args.Password) < 50 { - if zxcvbn.PasswordStrength(args.Password, nil).Score < requiredPassScore { - return errWeakPassword - } - } +// As per issue https://github.com/ava-labs/gecko/issues/195 it was found the longer the length of password the slower zxcvbn.PasswordStrength() performs. +// To avoid performance issues and DOS vector we only check the first 50 characters of the password. + checkPass := args.Password - if len(args.Password) >= 50 { - if zxcvbn.PasswordStrength(args.Password[:50], nil).Score < requiredPassScore { - return errWeakPassword - } - } + if len(args.Password) > 50 { + checkPass = args.Password[:50] + } + + if zxcvbn.PasswordStrength(checkPass, nil).Score < requiredPassScore { + return errWeakPassword + } usr := &User{} if err := usr.Initialize(args.Password); err != nil { From 311ce90977dc2d87fe3094f4c3e8171aace99d4e Mon Sep 17 00:00:00 2001 From: Shashank Date: Wed, 3 Jun 2020 19:47:39 +0530 Subject: [PATCH 04/42] Fixed go format --- api/keystore/service.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 72c7c24..e80b4d9 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -148,16 +148,16 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre return fmt.Errorf("user already exists: %s", args.Username) } -// As per issue https://github.com/ava-labs/gecko/issues/195 it was found the longer the length of password the slower zxcvbn.PasswordStrength() performs. -// To avoid performance issues and DOS vector we only check the first 50 characters of the password. + // As per issue https://github.com/ava-labs/gecko/issues/195 it was found the longer the length of password the slower zxcvbn.PasswordStrength() performs. + // To avoid performance issues and DOS vector we only check the first 50 characters of the password. checkPass := args.Password if len(args.Password) > 50 { - checkPass = args.Password[:50] + checkPass = args.Password[:50] } if zxcvbn.PasswordStrength(checkPass, nil).Score < requiredPassScore { - return errWeakPassword + return errWeakPassword } usr := &User{} From 9e74fdf15dcda89679115ca10a67bf03283af5bd Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sat, 6 Jun 2020 11:48:13 -0400 Subject: [PATCH 05/42] improve network logging --- network/network.go | 94 +++++++++++++++++++++---- snow/networking/router/subnet_router.go | 32 +++++---- 2 files changed, 97 insertions(+), 29 deletions(-) diff --git a/network/network.go b/network/network.go index 50471eb..83a9e99 100644 --- a/network/network.go +++ b/network/network.go @@ -12,6 +12,8 @@ import ( "sync/atomic" "time" + "github.com/ava-labs/gecko/utils/formatting" + "github.com/prometheus/client_golang/prometheus" "github.com/ava-labs/gecko/api/health" @@ -266,8 +268,11 @@ func (n *network) GetAcceptedFrontier(validatorIDs ids.ShortSet, chainID ids.ID, func (n *network) AcceptedFrontier(validatorID ids.ShortID, chainID ids.ID, requestID uint32, containerIDs ids.Set) { msg, err := n.b.AcceptedFrontier(chainID, requestID, containerIDs) if err != nil { - n.log.Error("attempted to pack too large of an AcceptedFrontier message.\nNumber of containerIDs: %d", - containerIDs.Len()) + n.log.Error("failed to build AcceptedFrontier(%s, %d, %s): %s", + chainID, + requestID, + containerIDs, + err) return // Packing message failed } @@ -279,7 +284,11 @@ func (n *network) AcceptedFrontier(validatorID ids.ShortID, chainID ids.ID, requ sent = peer.send(msg) } if !sent { - n.log.Debug("failed to send an AcceptedFrontier message to: %s", validatorID) + n.log.Debug("failed to send AcceptedFrontier(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerIDs) n.acceptedFrontier.numFailed.Inc() } else { n.acceptedFrontier.numSent.Inc() @@ -290,6 +299,11 @@ func (n *network) AcceptedFrontier(validatorID ids.ShortID, chainID ids.ID, requ func (n *network) GetAccepted(validatorIDs ids.ShortSet, chainID ids.ID, requestID uint32, containerIDs ids.Set) { msg, err := n.b.GetAccepted(chainID, requestID, containerIDs) if err != nil { + n.log.Error("failed to build GetAccepted(%s, %d, %s): %s", + chainID, + requestID, + containerIDs, + err) for _, validatorID := range validatorIDs.List() { vID := validatorID n.executor.Add(func() { n.router.GetAcceptedFailed(vID, chainID, requestID) }) @@ -307,6 +321,11 @@ func (n *network) GetAccepted(validatorIDs ids.ShortSet, chainID ids.ID, request sent = peer.send(msg) } if !sent { + n.log.Debug("failed to send GetAccepted(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerIDs) n.executor.Add(func() { n.router.GetAcceptedFailed(vID, chainID, requestID) }) n.getAccepted.numFailed.Inc() } else { @@ -319,8 +338,11 @@ func (n *network) GetAccepted(validatorIDs ids.ShortSet, chainID ids.ID, request func (n *network) Accepted(validatorID ids.ShortID, chainID ids.ID, requestID uint32, containerIDs ids.Set) { msg, err := n.b.Accepted(chainID, requestID, containerIDs) if err != nil { - n.log.Error("attempted to pack too large of an Accepted message.\nNumber of containerIDs: %d", - containerIDs.Len()) + n.log.Error("failed to build Accepted(%s, %d, %s): %s", + chainID, + requestID, + containerIDs, + err) return // Packing message failed } @@ -332,7 +354,11 @@ func (n *network) Accepted(validatorID ids.ShortID, chainID ids.ID, requestID ui sent = peer.send(msg) } if !sent { - n.log.Debug("failed to send an Accepted message to: %s", validatorID) + n.log.Debug("failed to send Accepted(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerIDs) n.accepted.numFailed.Inc() } else { n.accepted.numSent.Inc() @@ -352,7 +378,11 @@ func (n *network) Get(validatorID ids.ShortID, chainID ids.ID, requestID uint32, sent = peer.send(msg) } if !sent { - n.log.Debug("failed to send a Get message to: %s", validatorID) + n.log.Debug("failed to send Get(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerID) n.get.numFailed.Inc() } else { n.get.numSent.Inc() @@ -363,7 +393,12 @@ func (n *network) Get(validatorID ids.ShortID, chainID ids.ID, requestID uint32, func (n *network) Put(validatorID ids.ShortID, chainID ids.ID, requestID uint32, containerID ids.ID, container []byte) { msg, err := n.b.Put(chainID, requestID, containerID, container) if err != nil { - n.log.Error("failed to build Put message because of container of size %d", len(container)) + n.log.Error("failed to build Put(%s, %d, %s): %s. len(container) : %d", + chainID, + requestID, + containerID, + err, + len(container)) return } @@ -375,7 +410,12 @@ func (n *network) Put(validatorID ids.ShortID, chainID ids.ID, requestID uint32, sent = peer.send(msg) } if !sent { - n.log.Debug("failed to send a Put message to: %s", validatorID) + n.log.Debug("failed to send Put(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerID) + n.log.Verbo("container: %s", formatting.DumpBytes{Bytes: container}) n.put.numFailed.Inc() } else { n.put.numSent.Inc() @@ -390,7 +430,13 @@ func (n *network) PushQuery(validatorIDs ids.ShortSet, chainID ids.ID, requestID vID := validatorID n.executor.Add(func() { n.router.QueryFailed(vID, chainID, requestID) }) } - n.log.Error("attempted to pack too large of a PushQuery message.\nContainer length: %d", len(container)) + n.log.Error("failed to build PushQuery(%s, %d, %s): %s. len(container): %d", + chainID, + requestID, + containerID, + err, + len(container)) + n.log.Verbo("container: %s", formatting.DumpBytes{Bytes: container}) return // Packing message failed } @@ -404,7 +450,12 @@ func (n *network) PushQuery(validatorIDs ids.ShortSet, chainID ids.ID, requestID sent = peer.send(msg) } if !sent { - n.log.Debug("failed sending a PushQuery message to: %s", vID) + n.log.Debug("failed to send PushQuery(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerID) + n.log.Verbo("container: %s", formatting.DumpBytes{Bytes: container}) n.executor.Add(func() { n.router.QueryFailed(vID, chainID, requestID) }) n.pushQuery.numFailed.Inc() } else { @@ -428,7 +479,11 @@ func (n *network) PullQuery(validatorIDs ids.ShortSet, chainID ids.ID, requestID sent = peer.send(msg) } if !sent { - n.log.Debug("failed sending a PullQuery message to: %s", vID) + n.log.Debug("failed to send PullQuery(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + containerID) n.executor.Add(func() { n.router.QueryFailed(vID, chainID, requestID) }) n.pullQuery.numFailed.Inc() } else { @@ -441,7 +496,11 @@ func (n *network) PullQuery(validatorIDs ids.ShortSet, chainID ids.ID, requestID func (n *network) Chits(validatorID ids.ShortID, chainID ids.ID, requestID uint32, votes ids.Set) { msg, err := n.b.Chits(chainID, requestID, votes) if err != nil { - n.log.Error("failed to build Chits message because of %d votes", votes.Len()) + n.log.Error("failed to build Chits(%s, %d, %s): %s", + chainID, + requestID, + votes, + err) return } @@ -453,7 +512,11 @@ func (n *network) Chits(validatorID ids.ShortID, chainID ids.ID, requestID uint3 sent = peer.send(msg) } if !sent { - n.log.Debug("failed to send a Chits message to: %s", validatorID) + n.log.Debug("failed to send Chits(%s, %s, %d, %s)", + validatorID, + chainID, + requestID, + votes) n.chits.numFailed.Inc() } else { n.chits.numSent.Inc() @@ -463,7 +526,8 @@ func (n *network) Chits(validatorID ids.ShortID, chainID ids.ID, requestID uint3 // Gossip attempts to gossip the container to the network func (n *network) Gossip(chainID, containerID ids.ID, container []byte) { if err := n.gossipContainer(chainID, containerID, container); err != nil { - n.log.Error("error gossiping container %s to %s: %s", containerID, chainID, err) + n.log.Error("failed to Gossip(%s, %s): %s", chainID, containerID, err) + n.log.Verbo("container:\n%s", formatting.DumpBytes{Bytes: container}) } } diff --git a/snow/networking/router/subnet_router.go b/snow/networking/router/subnet_router.go index 36187dc..5ac7c80 100644 --- a/snow/networking/router/subnet_router.go +++ b/snow/networking/router/subnet_router.go @@ -7,6 +7,8 @@ import ( "sync" "time" + "github.com/ava-labs/gecko/utils/formatting" + "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow/networking/timeout" "github.com/ava-labs/gecko/utils/logging" @@ -81,7 +83,7 @@ func (sr *ChainRouter) RemoveChain(chainID ids.ID) { delete(sr.chains, chainID.Key()) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("can't remove unknown chain %s", chainID) } } @@ -95,7 +97,7 @@ func (sr *ChainRouter) GetAcceptedFrontier(validatorID ids.ShortID, chainID ids. if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetAcceptedFrontier(validatorID, requestID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("GetAcceptedFrontier(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } @@ -110,7 +112,7 @@ func (sr *ChainRouter) AcceptedFrontier(validatorID ids.ShortID, chainID ids.ID, if chain, exists := sr.chains[chainID.Key()]; exists { chain.AcceptedFrontier(validatorID, requestID, containerIDs) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("AcceptedFrontier(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerIDs) } } @@ -125,7 +127,7 @@ func (sr *ChainRouter) GetAcceptedFrontierFailed(validatorID ids.ShortID, chainI if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetAcceptedFrontierFailed(validatorID, requestID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("GetAcceptedFrontierFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } @@ -139,7 +141,7 @@ func (sr *ChainRouter) GetAccepted(validatorID ids.ShortID, chainID ids.ID, requ if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetAccepted(validatorID, requestID, containerIDs) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("GetAccepted(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerIDs) } } @@ -154,7 +156,7 @@ func (sr *ChainRouter) Accepted(validatorID ids.ShortID, chainID ids.ID, request if chain, exists := sr.chains[chainID.Key()]; exists { chain.Accepted(validatorID, requestID, containerIDs) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("Accepted(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerIDs) } } @@ -169,7 +171,7 @@ func (sr *ChainRouter) GetAcceptedFailed(validatorID ids.ShortID, chainID ids.ID if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetAcceptedFailed(validatorID, requestID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("GetAcceptedFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } @@ -182,7 +184,7 @@ func (sr *ChainRouter) Get(validatorID ids.ShortID, chainID ids.ID, requestID ui if chain, exists := sr.chains[chainID.Key()]; exists { chain.Get(validatorID, requestID, containerID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("Get(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerID) } } @@ -198,7 +200,8 @@ func (sr *ChainRouter) Put(validatorID ids.ShortID, chainID ids.ID, requestID ui if chain, exists := sr.chains[chainID.Key()]; exists { chain.Put(validatorID, requestID, containerID, container) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("Put(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerID) + sr.log.Verbo("container:\n%s", formatting.DumpBytes{Bytes: container}) } } @@ -212,7 +215,7 @@ func (sr *ChainRouter) GetFailed(validatorID ids.ShortID, chainID ids.ID, reques if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetFailed(validatorID, requestID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("GetFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } @@ -225,7 +228,8 @@ func (sr *ChainRouter) PushQuery(validatorID ids.ShortID, chainID ids.ID, reques if chain, exists := sr.chains[chainID.Key()]; exists { chain.PushQuery(validatorID, requestID, containerID, container) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("PushQuery(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerID) + sr.log.Verbo("container:\n%s", formatting.DumpBytes{Bytes: container}) } } @@ -238,7 +242,7 @@ func (sr *ChainRouter) PullQuery(validatorID ids.ShortID, chainID ids.ID, reques if chain, exists := sr.chains[chainID.Key()]; exists { chain.PullQuery(validatorID, requestID, containerID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("PullQuery(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, containerID) } } @@ -253,7 +257,7 @@ func (sr *ChainRouter) Chits(validatorID ids.ShortID, chainID ids.ID, requestID if chain, exists := sr.chains[chainID.Key()]; exists { chain.Chits(validatorID, requestID, votes) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("Chits(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID, votes) } } @@ -267,7 +271,7 @@ func (sr *ChainRouter) QueryFailed(validatorID ids.ShortID, chainID ids.ID, requ if chain, exists := sr.chains[chainID.Key()]; exists { chain.QueryFailed(validatorID, requestID) } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("QueryFailed(%s, %s, %d, %s) dropped due to unknown chain", validatorID, chainID, requestID) } } From c560eeab37a8fc906fd55c7a03bcf9e40c930dcb Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sat, 6 Jun 2020 11:50:46 -0400 Subject: [PATCH 06/42] lower/improve gossip logging --- network/network.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/network/network.go b/network/network.go index 83a9e99..d2d81fb 100644 --- a/network/network.go +++ b/network/network.go @@ -526,7 +526,7 @@ func (n *network) Chits(validatorID ids.ShortID, chainID ids.ID, requestID uint3 // Gossip attempts to gossip the container to the network func (n *network) Gossip(chainID, containerID ids.ID, container []byte) { if err := n.gossipContainer(chainID, containerID, container); err != nil { - n.log.Error("failed to Gossip(%s, %s): %s", chainID, containerID, err) + n.log.Debug("failed to Gossip(%s, %s): %s", chainID, containerID, err) n.log.Verbo("container:\n%s", formatting.DumpBytes{Bytes: container}) } } @@ -701,7 +701,9 @@ func (n *network) gossip() { } msg, err := n.b.PeerList(ips) if err != nil { - n.log.Warn("failed to gossip PeerList message due to %s", err) + n.log.Error("failed to build peer list to gossip: %s. len(ips): %d", + err, + len(ips)) continue } From b576f273976b86f6e3ea84e55f3af6463c2bd893 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 10 Jun 2020 16:20:40 -0400 Subject: [PATCH 07/42] comments/reorganize --- vms/components/codec/codec.go | 95 ++++++++++++++++++------------ vms/components/codec/codec_test.go | 4 +- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 72192cb..5896464 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -4,8 +4,10 @@ package codec import ( + "encoding/binary" "errors" "fmt" + "math" "reflect" "unicode" @@ -15,14 +17,14 @@ import ( const ( defaultMaxSize = 1 << 18 // default max size, in bytes, of something being marshalled by Marshal() defaultMaxSliceLength = 1 << 18 // default max length of a slice being marshalled by Marshal() + maxStringLen = math.MaxInt16 ) // ErrBadCodec is returned when one tries to perform an operation // using an unknown codec var ( errBadCodec = errors.New("wrong or unknown codec used") - errNil = errors.New("can't marshal nil value") - errUnmarshalNil = errors.New("can't unmarshal into nil") + errNil = errors.New("can't marshal/unmarshal nil value") errNeedPointer = errors.New("must unmarshal into a pointer") errMarshalUnregisteredType = errors.New("can't marshal an unregistered type") errUnmarshalUnregisteredType = errors.New("can't unmarshal an unregistered type") @@ -62,7 +64,7 @@ func New(maxSize, maxSliceLen int) Codec { // NewDefault returns a new codec with reasonable default values func NewDefault() Codec { return New(defaultMaxSize, defaultMaxSliceLength) } -// RegisterType is used to register types that may be unmarshaled into an interface typed value +// RegisterType is used to register types that may be unmarshaled into an interface // [val] is a value of the type being registered func (c codec) RegisterType(val interface{}) error { valType := reflect.TypeOf(val) @@ -79,23 +81,20 @@ func (c codec) RegisterType(val interface{}) error { // 2) We use "marshal" and "serialize" interchangeably, and "unmarshal" and "deserialize" interchangeably // 3) To include a field of a struct in the serialized form, add the tag `serialize:"true"` to it // 4) These typed members of a struct may be serialized: -// bool, string, uint[8,16,32,64, int[8,16,32,64], +// bool, string, uint[8,16,32,64], int[8,16,32,64], // structs, slices, arrays, interface. -// structs, slices and arrays can only be serialized if their constituent parts can be. -// 5) To marshal an interface typed value, you must pass a _pointer_ to the value -// 6) If you want to be able to unmarshal into an interface typed value, -// you must call codec.RegisterType([instance of the type that fulfills the interface]). +// structs, slices and arrays can only be serialized if their constituent values can be. +// 5) To marshal an interface, you must pass a pointer to the value +// 6) To unmarshal an interface, you must call codec.RegisterType([instance of the type that fulfills the interface]). // 7) nil slices will be unmarshaled as an empty slice of the appropriate type // 8) Serialized fields must be exported // Marshal returns the byte representation of [value] -// If you want to marshal an interface, [value] must be a pointer -// to the interface +// To marshal an interface, [value] must be a pointer to the interface func (c codec) Marshal(value interface{}) ([]byte, error) { if value == nil { return nil, errNil } - return c.marshal(reflect.ValueOf(value)) } @@ -105,46 +104,69 @@ func (c codec) marshal(value reflect.Value) ([]byte, error) { t := value.Type() valueKind := value.Kind() + + // Case: Value can't be marshalled switch valueKind { - case reflect.Interface, reflect.Ptr, reflect.Slice: - if value.IsNil() { + case reflect.Interface, reflect.Ptr, reflect.Slice, reflect.Invalid: + if value.IsNil() { // Can't marshal nil return nil, errNil } } + // Case: Value is of known size; return its byte repr. switch valueKind { case reflect.Uint8: - p.PackByte(uint8(value.Uint())) - return p.Bytes, p.Err + return []byte{byte(value.Uint())}, nil case reflect.Int8: - p.PackByte(uint8(value.Int())) - return p.Bytes, p.Err + return []byte{byte(value.Int())}, nil case reflect.Uint16: - p.PackShort(uint16(value.Uint())) - return p.Bytes, p.Err + bytes := make([]byte, 2, 2) + binary.BigEndian.PutUint16(bytes, uint16(value.Uint())) + return bytes, nil case reflect.Int16: - p.PackShort(uint16(value.Int())) - return p.Bytes, p.Err + bytes := make([]byte, 2, 2) + binary.BigEndian.PutUint16(bytes, uint16(value.Int())) + return bytes, nil case reflect.Uint32: - p.PackInt(uint32(value.Uint())) - return p.Bytes, p.Err + bytes := make([]byte, 4, 4) + binary.BigEndian.PutUint32(bytes, uint32(value.Uint())) + return bytes, nil case reflect.Int32: - p.PackInt(uint32(value.Int())) - return p.Bytes, p.Err + bytes := make([]byte, 4, 4) + binary.BigEndian.PutUint32(bytes, uint32(value.Int())) + return bytes, nil case reflect.Uint64: - p.PackLong(value.Uint()) - return p.Bytes, p.Err + bytes := make([]byte, 8, 8) + binary.BigEndian.PutUint64(bytes, uint64(value.Uint())) + return bytes, nil case reflect.Int64: - p.PackLong(uint64(value.Int())) - return p.Bytes, p.Err + bytes := make([]byte, 8, 8) + binary.BigEndian.PutUint64(bytes, uint64(value.Int())) + return bytes, nil case reflect.Uintptr, reflect.Ptr: return c.marshal(value.Elem()) case reflect.String: - p.PackStr(value.String()) - return p.Bytes, p.Err + asStr := value.String() + strSize := len(asStr) + if strSize > maxStringLen { + return nil, errSliceTooLarge + } + bytes := make([]byte, 2+strSize, 2+strSize) + binary.BigEndian.PutUint16(bytes[0:2], uint16(strSize)) + if strSize == 0 { + return bytes, nil + } + copy(bytes[2:], []byte(asStr)) + return bytes, nil case reflect.Bool: - p.PackBool(value.Bool()) - return p.Bytes, p.Err + if value.Bool() { + return []byte{1}, nil + } + return []byte{0}, nil + } + + // Case: Value is of unknown size. Calculate its size and fill byte array. + switch valueKind { case reflect.Interface: typeID, ok := c.typeToTypeID[reflect.TypeOf(value.Interface())] // Get the type ID of the value being marshaled if !ok { @@ -181,7 +203,7 @@ func (c codec) marshal(value reflect.Value) ([]byte, error) { continue } if unicode.IsLower(rune(field.Name[0])) { // Can only marshal exported fields - return nil, errMarshalUnexportedField + return nil, fmt.Errorf("can't marshal exported field %s", field.Name) } fieldVal := value.Field(i) // The field we're serializing if fieldVal.Kind() == reflect.Slice && fieldVal.IsNil() { @@ -195,8 +217,7 @@ func (c codec) marshal(value reflect.Value) ([]byte, error) { p.PackFixedBytes(fieldBytes) } return p.Bytes, p.Err - case reflect.Invalid: - return nil, errUnmarshalNil + default: return nil, errUnknownType } @@ -332,7 +353,7 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { // Assign to the top-level struct's member field.Set(underlyingValue) case reflect.Invalid: - return errUnmarshalNil + return errNil default: return errUnknownType } diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index 6fdfeba..a58f685 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -407,8 +407,8 @@ func TestSerializeUnexportedField(t *testing.T) { } codec := NewDefault() - if _, err := codec.Marshal(myS); err != errMarshalUnexportedField { - t.Fatalf("expected err to be errUnexportedField but was %v", err) + if _, err := codec.Marshal(myS); err == nil { + t.Fatalf("expected err but got none") } } From e073b4e8ad99cc026e32d9ce34f320bf9d647892 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Thu, 11 Jun 2020 17:01:28 -0400 Subject: [PATCH 08/42] Prevent duplicated addresses in avm import key --- vms/avm/service.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/vms/avm/service.go b/vms/avm/service.go index f71d607..1dce66d 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -665,14 +665,26 @@ func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *I return fmt.Errorf("problem saving key %w", err) } - addresses, _ := user.Addresses(db) - addresses = append(addresses, sk.PublicKey().Address()) + addresses, err := user.Addresses(db) + if err != nil { + return fmt.Errorf("problem saving key while getting existing addresses: %w", err) + } + newAddress := sk.PublicKey().Address() + exists := false + for _, address := range addresses { + if newAddress.Equals(address) { + exists = true + } + } + if !exists { + addresses = append(addresses, newAddress) - if err := user.SetAddresses(db, addresses); err != nil { - return fmt.Errorf("problem saving addresses: %w", err) + if err := user.SetAddresses(db, addresses); err != nil { + return fmt.Errorf("problem saving addresses: %w", err) + } } - reply.Address = service.vm.Format(sk.PublicKey().Address().Bytes()) + reply.Address = service.vm.Format(newAddress.Bytes()) return nil } From 7879dd17688872a6260a60b6ace13e4e7dc702d6 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Thu, 11 Jun 2020 18:16:21 -0400 Subject: [PATCH 09/42] upgrade codec to be more efficient. Passes all codec tests. Failing some other tests due to new format --- vms/components/codec/codec.go | 541 ++++++++++++++++++++++------- vms/components/codec/codec_test.go | 72 ++-- 2 files changed, 460 insertions(+), 153 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 5896464..ad09964 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -10,8 +10,6 @@ import ( "math" "reflect" "unicode" - - "github.com/ava-labs/gecko/utils/wrappers" ) const ( @@ -89,275 +87,552 @@ func (c codec) RegisterType(val interface{}) error { // 7) nil slices will be unmarshaled as an empty slice of the appropriate type // 8) Serialized fields must be exported -// Marshal returns the byte representation of [value] // To marshal an interface, [value] must be a pointer to the interface func (c codec) Marshal(value interface{}) ([]byte, error) { if value == nil { return nil, errNil } - return c.marshal(reflect.ValueOf(value)) + size, f, err := c.marshal(reflect.ValueOf(value)) + if err != nil { + return nil, err + } + + bytes := make([]byte, size, size) + if err := f(bytes); err != nil { + return nil, err + } + return bytes, nil } -// Marshal [value] to bytes -func (c codec) marshal(value reflect.Value) ([]byte, error) { - p := wrappers.Packer{MaxSize: c.maxSize, Bytes: []byte{}} - t := value.Type() - +// marshal returns: +// 1) The size, in bytes, of the byte representation of [value] +// 2) A slice of functions, where each function writes bytes to its argument +// and returns the number of bytes it wrote. +// When these functions are called in order, they write [value] to a byte slice. +// 3) An error +func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err error) { valueKind := value.Kind() // Case: Value can't be marshalled switch valueKind { - case reflect.Interface, reflect.Ptr, reflect.Slice, reflect.Invalid: - if value.IsNil() { // Can't marshal nil - return nil, errNil + case reflect.Interface, reflect.Ptr, reflect.Invalid: + if value.IsNil() { // Can't marshal nil or nil pointers + return 0, nil, errNil } } // Case: Value is of known size; return its byte repr. switch valueKind { case reflect.Uint8: - return []byte{byte(value.Uint())}, nil + size = 1 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 1 { + return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + copy(b, []byte{byte(value.Uint())}) + return nil + } + return case reflect.Int8: - return []byte{byte(value.Int())}, nil + size = 1 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 1 { + return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + copy(b, []byte{byte(value.Int())}) + return nil + } + return case reflect.Uint16: - bytes := make([]byte, 2, 2) - binary.BigEndian.PutUint16(bytes, uint16(value.Uint())) - return bytes, nil + size = 2 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 2 { + return fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + } + binary.BigEndian.PutUint16(b, uint16(value.Uint())) + return nil + } + return case reflect.Int16: - bytes := make([]byte, 2, 2) - binary.BigEndian.PutUint16(bytes, uint16(value.Int())) - return bytes, nil + size = 2 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 2 { + return fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + } + binary.BigEndian.PutUint16(b, uint16(value.Int())) + return nil + } + return case reflect.Uint32: - bytes := make([]byte, 4, 4) - binary.BigEndian.PutUint32(bytes, uint32(value.Uint())) - return bytes, nil + size = 4 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 4 { + return fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + } + binary.BigEndian.PutUint32(b, uint32(value.Uint())) + return nil + } + return case reflect.Int32: - bytes := make([]byte, 4, 4) - binary.BigEndian.PutUint32(bytes, uint32(value.Int())) - return bytes, nil + size = 4 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 4 { + return fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + } + binary.BigEndian.PutUint32(b, uint32(value.Int())) + return nil + } + return case reflect.Uint64: - bytes := make([]byte, 8, 8) - binary.BigEndian.PutUint64(bytes, uint64(value.Uint())) - return bytes, nil + size = 8 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 8 { + return fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + } + binary.BigEndian.PutUint64(b, uint64(value.Uint())) + return nil + } + return case reflect.Int64: - bytes := make([]byte, 8, 8) - binary.BigEndian.PutUint64(bytes, uint64(value.Int())) - return bytes, nil - case reflect.Uintptr, reflect.Ptr: - return c.marshal(value.Elem()) + size = 8 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 8 { + return fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + } + binary.BigEndian.PutUint64(b, uint64(value.Int())) + return nil + } + return case reflect.String: asStr := value.String() strSize := len(asStr) if strSize > maxStringLen { - return nil, errSliceTooLarge + return 0, nil, errSliceTooLarge } - bytes := make([]byte, 2+strSize, 2+strSize) - binary.BigEndian.PutUint16(bytes[0:2], uint16(strSize)) - if strSize == 0 { - return bytes, nil - } - copy(bytes[2:], []byte(asStr)) - return bytes, nil - case reflect.Bool: - if value.Bool() { - return []byte{1}, nil - } - return []byte{0}, nil - } - // Case: Value is of unknown size. Calculate its size and fill byte array. - switch valueKind { + size = strSize + 2 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < size { + return fmt.Errorf("expected len(bytes) to be at least %d but is %d", size, bytesLen) + } + binary.BigEndian.PutUint16(b, uint16(strSize)) + if strSize == 0 { + return nil + } + copy(b[2:], []byte(asStr)) + return nil + } + return + case reflect.Bool: + size = 1 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 1 { + return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + if value.Bool() { + copy(b, []byte{1}) + } else { + copy(b, []byte{0}) + } + return nil + } + return + case reflect.Uintptr, reflect.Ptr: + return c.marshal(value.Elem()) case reflect.Interface: typeID, ok := c.typeToTypeID[reflect.TypeOf(value.Interface())] // Get the type ID of the value being marshaled if !ok { - return nil, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(value.Interface()).String()) + return 0, nil, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(value.Interface()).String()) } - p.PackInt(typeID) - bytes, err := c.Marshal(value.Interface()) - if err != nil { - return nil, err + + subsize, subfunc, subErr := c.marshal(reflect.ValueOf(value.Interface())) // TODO: Is this right? + if subErr != nil { + return 0, nil, subErr } - p.PackFixedBytes(bytes) - if p.Errored() { - return nil, p.Err - } - return p.Bytes, err - case reflect.Array, reflect.Slice: - numElts := value.Len() // # elements in the slice/array (assumed to be <= 2^31 - 1) - // If this is a slice, pack the number of elements in the slice - if valueKind == reflect.Slice { - p.PackInt(uint32(numElts)) - } - for i := 0; i < numElts; i++ { // Pack each element in the slice/array - eltBytes, err := c.marshal(value.Index(i)) - if err != nil { - return nil, err + + size = 4 + subsize // 4 because we pack the type ID, a uint32 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 4+subsize { + return fmt.Errorf("expected len(bytes) to be at least %d but is %d", 4+subsize, bytesLen) } - p.PackFixedBytes(eltBytes) + binary.BigEndian.PutUint32(b, uint32(typeID)) + if len(b) == 4 { + return nil + } + return subfunc(b[4:]) } - return p.Bytes, p.Err + return + case reflect.Slice: + if value.IsNil() { + size = 1 + f = func(b []byte) error { + if bytesLen := len(b); bytesLen < 1 { + return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + b[0] = 1 // slice is nil; set isNil flag to 1 + return nil + } + return + } + + numElts := value.Len() // # elements in the slice/array (assumed to be <= 2^31 - 1) + if numElts > c.maxSliceLen { + return 0, nil, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) + } + + size = 5 // 1 for the isNil flag. 0 --> this slice isn't nil. 1--> it is nil. + // 4 for the size of the slice (uint32) + + // offsets[i] is the index in the byte array that subFuncs[i] will start writing at + offsets := make([]int, numElts+1, numElts+1) + if numElts != 0 { + offsets[1] = 5 // 1 for nil flag, 4 for slice size + } + subFuncs := make([]func([]byte) error, numElts+1, numElts+1) + subFuncs[0] = func(b []byte) error { // write the nil flag and number of elements + if bytesLen := len(b); bytesLen < 5 { + return fmt.Errorf("expected len(bytes) to be at least 5 but is %d", bytesLen) + } + b[0] = 0 // slice is non-nil; set isNil flag to 0 + binary.BigEndian.PutUint32(b[1:], uint32(numElts)) + return nil + } + for i := 1; i < numElts+1; i++ { // Process each element in the slice + subSize, subFunc, subErr := c.marshal(value.Index(i - 1)) + if subErr != nil { + return 0, nil, subErr + } + size += subSize + if i != numElts { // set offest for next function unless this is last ieration + offsets[i+1] = offsets[i] + subSize + } + subFuncs[i] = subFunc + } + + if subFuncsLen := len(subFuncs); subFuncsLen != len(offsets) { + return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) + } + + f = func(b []byte) error { + bytesLen := len(b) + for i, f := range subFuncs { + offset := offsets[i] + if offset > bytesLen { + return fmt.Errorf("attempted out of bounds slice. offset: %d. bytesLen: %d", offset, bytesLen) + } + if err := f(b[offset:]); err != nil { + return err + } + } + return nil + } + return + case reflect.Array: + numElts := value.Len() // # elements in the slice/array (assumed to be <= 2^31 - 1) + if numElts > math.MaxUint32 { + return 0, nil, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) + } + + size = 0 + // offsets[i] is the index in the byte array that subFuncs[i] will start writing at + offsets := make([]int, numElts, numElts) + offsets[1] = 4 // 4 for slice size + subFuncs := make([]func([]byte) error, numElts, numElts) + for i := 0; i < numElts; i++ { // Process each element in the array + subSize, subFunc, subErr := c.marshal(value.Index(i)) + if subErr != nil { + return 0, nil, subErr + } + size += subSize + if i != numElts-1 { // set offest for next function unless this is last ieration + offsets[i+1] = offsets[i] + subSize + } + subFuncs[i] = subFunc + } + + if subFuncsLen := len(subFuncs); subFuncsLen != len(offsets) { + return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) + } + + f = func(b []byte) error { + bytesLen := len(b) + for i, f := range subFuncs { + offset := offsets[i] + if offset > bytesLen { + return fmt.Errorf("attempted out of bounds slice") + } + if err := f(b[offset:]); err != nil { + return err + } + } + return nil + } + return case reflect.Struct: - for i := 0; i < t.NumField(); i++ { // Go through all fields of this struct + t := value.Type() + numFields := t.NumField() + size = 0 + // offsets[i] is the index in the byte array that subFuncs[i] will start writing at + offsets := make([]int, 0, numFields) + offsets = append(offsets, 0) + subFuncs := make([]func([]byte) error, 0, numFields) + for i := 0; i < numFields; i++ { // Go through all fields of this struct field := t.Field(i) if !shouldSerialize(field) { // Skip fields we don't need to serialize continue } if unicode.IsLower(rune(field.Name[0])) { // Can only marshal exported fields - return nil, fmt.Errorf("can't marshal exported field %s", field.Name) + return 0, nil, fmt.Errorf("can't marshal unexported field %s", field.Name) } - fieldVal := value.Field(i) // The field we're serializing - if fieldVal.Kind() == reflect.Slice && fieldVal.IsNil() { - p.PackInt(0) - continue - } - fieldBytes, err := c.marshal(fieldVal) // Serialize the field + fieldVal := value.Field(i) // The field we're serializing + subSize, subfunc, err := c.marshal(fieldVal) // Serialize the field if err != nil { - return nil, err + return 0, nil, err + } + size += subSize + subFuncs = append(subFuncs, subfunc) + if i != numFields-1 { // set offset for next function if not last iteration + offsets = append(offsets, offsets[len(offsets)-1]+subSize) } - p.PackFixedBytes(fieldBytes) } - return p.Bytes, p.Err + f = func(b []byte) error { + bytesLen := len(b) + for i, f := range subFuncs { + offset := offsets[i] + if offset > bytesLen { + return fmt.Errorf("attempted out of bounds slice") + } + if err := f(b[offset:]); err != nil { + return err + } + } + return nil + } + return default: - return nil, errUnknownType + return 0, nil, errUnknownType } } // Unmarshal unmarshals [bytes] into [dest], where // [dest] must be a pointer or interface func (c codec) Unmarshal(bytes []byte, dest interface{}) error { - p := &wrappers.Packer{Bytes: bytes} - - if len(bytes) > c.maxSize { + switch { + case len(bytes) > c.maxSize: return errSliceTooLarge - } - - if dest == nil { + case dest == nil: return errNil } destPtr := reflect.ValueOf(dest) - if destPtr.Kind() != reflect.Ptr { return errNeedPointer } destVal := destPtr.Elem() - - err := c.unmarshal(p, destVal) + bytesRead, err := c.unmarshal(bytes, destVal) if err != nil { return err } - if p.Offset != len(p.Bytes) { - return fmt.Errorf("has %d leftover bytes after unmarshalling", len(p.Bytes)-p.Offset) + if l := len(bytes); l != bytesRead { + return fmt.Errorf("%d leftover bytes after unmarshalling", l-bytesRead) } return nil } -// Unmarshal bytes from [p] into [field] +// Unmarshal bytes from [bytes] into [field] // [field] must be addressable -func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { +// Returns the number of bytes read from [bytes] +func (c codec) unmarshal(bytes []byte, field reflect.Value) (int, error) { + bytesLen := len(bytes) kind := field.Kind() switch kind { case reflect.Uint8: - field.SetUint(uint64(p.UnpackByte())) + if bytesLen < 1 { + return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + field.SetUint(uint64(bytes[0])) + return 1, nil case reflect.Int8: - field.SetInt(int64(p.UnpackByte())) + if bytesLen < 1 { + return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + field.SetInt(int64(bytes[0])) + return 1, nil case reflect.Uint16: - field.SetUint(uint64(p.UnpackShort())) + if bytesLen < 2 { + return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + } + field.SetUint(uint64(binary.BigEndian.Uint16(bytes))) + return 2, nil case reflect.Int16: - field.SetInt(int64(p.UnpackShort())) + if bytesLen < 2 { + return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + } + field.SetInt(int64(binary.BigEndian.Uint16(bytes))) + return 2, nil case reflect.Uint32: - field.SetUint(uint64(p.UnpackInt())) + if bytesLen < 4 { + return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + } + field.SetUint(uint64(binary.BigEndian.Uint32(bytes))) + return 4, nil case reflect.Int32: - field.SetInt(int64(p.UnpackInt())) + if bytesLen < 4 { + return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + } + field.SetInt(int64(binary.BigEndian.Uint32(bytes))) + return 4, nil case reflect.Uint64: - field.SetUint(p.UnpackLong()) + if bytesLen < 4 { + return 0, fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + } + field.SetUint(uint64(binary.BigEndian.Uint64(bytes))) + return 8, nil case reflect.Int64: - field.SetInt(int64(p.UnpackLong())) + if bytesLen < 4 { + return 0, fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + } + field.SetInt(int64(binary.BigEndian.Uint64(bytes))) + return 8, nil case reflect.Bool: - field.SetBool(p.UnpackBool()) + if bytesLen < 1 { + return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + if bytes[0] == 0 { + field.SetBool(false) + } else { + field.SetBool(true) + } + return 1, nil case reflect.Slice: - sliceLen := int(p.UnpackInt()) // number of elements in the slice - if sliceLen < 0 || sliceLen > c.maxSliceLen { - return errSliceTooLarge + if bytesLen < 1 { + return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + } + if bytes[0] == 1 { // isNil flag is 1 --> this slice is nil + return 1, nil } - // First set [field] to be a slice of the appropriate type/capacity (right now [field] is nil) - slice := reflect.MakeSlice(field.Type(), sliceLen, sliceLen) + numElts := int(binary.BigEndian.Uint32(bytes[1:])) // number of elements in the slice + if numElts > c.maxSliceLen { + return 0, fmt.Errorf("slice length, %d, exceeds maximum, %d", numElts, c.maxSliceLen) + } + + // set [field] to be a slice of the appropriate type/capacity (right now [field] is nil) + slice := reflect.MakeSlice(field.Type(), numElts, numElts) field.Set(slice) + // Unmarshal each element into the appropriate index of the slice - for i := 0; i < sliceLen; i++ { - if err := c.unmarshal(p, field.Index(i)); err != nil { - return err + bytesRead := 5 // 1 for isNil flag, 4 for numElts + for i := 0; i < numElts; i++ { + if bytesRead > bytesLen { + return 0, fmt.Errorf("attempted out of bounds slice") } + n, err := c.unmarshal(bytes[bytesRead:], field.Index(i)) + if err != nil { + return 0, err + } + bytesRead += n } + return bytesRead, nil case reflect.Array: + bytesRead := 0 for i := 0; i < field.Len(); i++ { - if err := c.unmarshal(p, field.Index(i)); err != nil { - return err + if bytesRead > bytesLen { + return 0, fmt.Errorf("attempted out of bounds slice") } + n, err := c.unmarshal(bytes[bytesRead:], field.Index(i)) + if err != nil { + return 0, err + } + bytesRead += n } + return bytesRead, nil case reflect.String: - field.SetString(p.UnpackStr()) + if bytesLen < 2 { + return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + } + strLen := int(binary.BigEndian.Uint16(bytes)) + if bytesLen < 2+strLen { + return 0, fmt.Errorf("expected len(bytes) to be at least %d but is %d", 2+strLen, bytesLen) + } + if strLen > 0 { + field.SetString(string(bytes[2 : 2+strLen])) + } else { + field.SetString("") + } + return strLen + 2, nil case reflect.Interface: + if bytesLen < 4 { + return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + } + // Get the type ID - typeID := p.UnpackInt() + typeID := binary.BigEndian.Uint32(bytes) // Get a struct that implements the interface typ, ok := c.typeIDToType[typeID] if !ok { - return errUnmarshalUnregisteredType + return 0, errUnmarshalUnregisteredType } // Ensure struct actually does implement the interface fieldType := field.Type() if !typ.Implements(fieldType) { - return fmt.Errorf("%s does not implement interface %s", typ, fieldType) + return 0, fmt.Errorf("%s does not implement interface %s", typ, fieldType) } concreteInstancePtr := reflect.New(typ) // instance of the proper type // Unmarshal into the struct - if err := c.unmarshal(p, concreteInstancePtr.Elem()); err != nil { - return err + + n, err := c.unmarshal(bytes[4:], concreteInstancePtr.Elem()) + if err != nil { + return 0, err } // And assign the filled struct to the field field.Set(concreteInstancePtr.Elem()) + return n + 4, nil case reflect.Struct: // Type of this struct structType := reflect.TypeOf(field.Interface()) // Go through all the fields and umarshal into each + bytesRead := 0 for i := 0; i < structType.NumField(); i++ { structField := structType.Field(i) if !shouldSerialize(structField) { // Skip fields we don't need to unmarshal continue } if unicode.IsLower(rune(structField.Name[0])) { // Only unmarshal into exported field - return errUnmarshalUnexportedField + return 0, errUnmarshalUnexportedField } - field := field.Field(i) // Get the field - if err := c.unmarshal(p, field); err != nil { // Unmarshal into the field - return err + field := field.Field(i) // Get the field + if bytesRead > bytesLen { + return 0, fmt.Errorf("attempted out of bounds slice") } - if p.Errored() { // If there was an error just return immediately - return p.Err + n, err := c.unmarshal(bytes[bytesRead:], field) // Unmarshal into the field + if err != nil { + return 0, err } + bytesRead += n } + return bytesRead, nil case reflect.Ptr: // Get the type this pointer points to underlyingType := field.Type().Elem() // Create a new pointer to a new value of the underlying type underlyingValue := reflect.New(underlyingType) // Fill the value - if err := c.unmarshal(p, underlyingValue.Elem()); err != nil { - return err + n, err := c.unmarshal(bytes, underlyingValue.Elem()) + if err != nil { + return 0, err } // Assign to the top-level struct's member field.Set(underlyingValue) + return n, nil case reflect.Invalid: - return errNil + return 0, errNil default: - return errUnknownType + return 0, errUnknownType } - return p.Err } // Returns true iff [field] should be serialized diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index a58f685..7100a79 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -97,6 +97,7 @@ func TestStruct(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("myStructBytes: %v", myStructBytes) myStructUnmarshaled := &myStruct{} err = codec.Unmarshal(myStructBytes, myStructUnmarshaled) @@ -370,7 +371,7 @@ func TestString(t *testing.T) { } } -// Ensure a nil slice is unmarshaled as an empty slice +// Ensure a nil slice is marshaled/unmarshaled correctly func TestNilSlice(t *testing.T) { type structWithSlice struct { Slice []byte `serialize:"true"` @@ -388,13 +389,13 @@ func TestNilSlice(t *testing.T) { t.Fatal(err) } - if structUnmarshaled.Slice == nil || len(structUnmarshaled.Slice) != 0 { - t.Fatal("expected slice to be empty slice") + if structUnmarshaled.Slice != nil { + t.Fatal("expected slice to be nil") } } // Ensure that trying to serialize a struct with an unexported member -// that has `serialize:"true"` returns errUnexportedField +// that has `serialize:"true"` returns error func TestSerializeUnexportedField(t *testing.T) { type s struct { ExportedField string `serialize:"true"` @@ -426,12 +427,12 @@ func TestSerializeOfNoSerializeField(t *testing.T) { codec := NewDefault() marshalled, err := codec.Marshal(myS) if err != nil { - t.Fatalf("Unexpected error %q", err) + t.Fatal(err) } unmarshalled := s{} err = codec.Unmarshal(marshalled, &unmarshalled) if err != nil { - t.Fatalf("Unexpected error %q", err) + t.Fatal(err) } expectedUnmarshalled := s{SerializedField: "Serialize me"} if !reflect.DeepEqual(unmarshalled, expectedUnmarshalled) { @@ -443,11 +444,12 @@ type simpleSliceStruct struct { Arr []uint32 `serialize:"true"` } -func TestEmptySliceSerialization(t *testing.T) { +// Test marshalling of nil slice +func TestNilSliceSerialization(t *testing.T) { codec := NewDefault() val := &simpleSliceStruct{} - expected := []byte{0, 0, 0, 0} + expected := []byte{1} // 1 for isNil result, err := codec.Marshal(val) if err != nil { t.Fatal(err) @@ -456,6 +458,40 @@ func TestEmptySliceSerialization(t *testing.T) { if !bytes.Equal(expected, result) { t.Fatalf("\nExpected: 0x%x\nResult: 0x%x", expected, result) } + + valUnmarshaled := &simpleSliceStruct{} + if err = codec.Unmarshal(result, &valUnmarshaled); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(valUnmarshaled, val) { + t.Logf("val: %v\n", val) + t.Logf("valUnmarshaled: %v\n", valUnmarshaled) + t.Fatal("should be same") + } +} + +// Test marshaling a slice that has 0 elements (but isn't nil) +func TestEmptySliceSerialization(t *testing.T) { + codec := NewDefault() + + val := &simpleSliceStruct{Arr: make([]uint32, 0, 1)} + expected := []byte{0, 0, 0, 0, 0} // 0 for isNil flag, 0 for size + result, err := codec.Marshal(val) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(expected, result) { + t.Fatalf("\nExpected: 0x%x\nResult: 0x%x", expected, result) + } + + valUnmarshaled := &simpleSliceStruct{} + if err = codec.Unmarshal(result, &valUnmarshaled); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(valUnmarshaled, val) { + t.Logf("val: %v\n", val) + t.Logf("valUnmarshaled: %v\n", valUnmarshaled) + t.Fatal("should be same") + } } type emptyStruct struct{} @@ -464,13 +500,14 @@ type nestedSliceStruct struct { Arr []emptyStruct `serialize:"true"` } +// Test marshaling slice that is not nil and not empty func TestSliceWithEmptySerialization(t *testing.T) { codec := NewDefault() val := &nestedSliceStruct{ Arr: make([]emptyStruct, 1000), } - expected := []byte{0x00, 0x00, 0x03, 0xE8} + expected := []byte{0x00, 0x00, 0x00, 0x03, 0xE8} // 0 for isNil flag, then 1000 for numElts result, err := codec.Marshal(val) if err != nil { t.Fatal(err) @@ -485,7 +522,7 @@ func TestSliceWithEmptySerialization(t *testing.T) { t.Fatal(err) } if len(unmarshaled.Arr) != 1000 { - t.Fatalf("Should have created an array of length %d", 1000) + t.Fatalf("Should have created a slice of length %d", 1000) } } @@ -493,20 +530,15 @@ func TestSliceWithEmptySerializationOutOfMemory(t *testing.T) { codec := NewDefault() val := &nestedSliceStruct{ - Arr: make([]emptyStruct, 1000000), + Arr: make([]emptyStruct, defaultMaxSliceLength+1), } - expected := []byte{0x00, 0x0f, 0x42, 0x40} // 1,000,000 in hex - result, err := codec.Marshal(val) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(expected, result) { - t.Fatalf("\nExpected: 0x%x\nResult: 0x%x", expected, result) + bytes, err := codec.Marshal(val) + if err == nil { + t.Fatal("should have failed due to slice length too large") } unmarshaled := nestedSliceStruct{} - if err := codec.Unmarshal(expected, &unmarshaled); err == nil { + if err := codec.Unmarshal(bytes, &unmarshaled); err == nil { t.Fatalf("Should have errored due to excess memory requested") } } From 617a1580974e84f32a285e2fe7c03d7756cab5f9 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 12 Jun 2020 10:41:02 -0400 Subject: [PATCH 10/42] use wrappers.packer instead of byte array --- vms/components/codec/codec.go | 401 +++++++++++------------------ vms/components/codec/codec_test.go | 4 +- 2 files changed, 154 insertions(+), 251 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index ad09964..873aac2 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -4,18 +4,19 @@ package codec import ( - "encoding/binary" "errors" "fmt" "math" "reflect" "unicode" + + "github.com/ava-labs/gecko/utils/wrappers" ) const ( defaultMaxSize = 1 << 18 // default max size, in bytes, of something being marshalled by Marshal() defaultMaxSliceLength = 1 << 18 // default max length of a slice being marshalled by Marshal() - maxStringLen = math.MaxInt16 + maxStringLen = math.MaxUint16 ) // ErrBadCodec is returned when one tries to perform an operation @@ -84,8 +85,7 @@ func (c codec) RegisterType(val interface{}) error { // structs, slices and arrays can only be serialized if their constituent values can be. // 5) To marshal an interface, you must pass a pointer to the value // 6) To unmarshal an interface, you must call codec.RegisterType([instance of the type that fulfills the interface]). -// 7) nil slices will be unmarshaled as an empty slice of the appropriate type -// 8) Serialized fields must be exported +// 7) Serialized fields must be exported // To marshal an interface, [value] must be a pointer to the interface func (c codec) Marshal(value interface{}) ([]byte, error) { @@ -97,11 +97,11 @@ func (c codec) Marshal(value interface{}) ([]byte, error) { return nil, err } - bytes := make([]byte, size, size) - if err := f(bytes); err != nil { + p := &wrappers.Packer{MaxSize: size, Bytes: make([]byte, 0, size)} + if err := f(p); err != nil { return nil, err } - return bytes, nil + return p.Bytes, nil } // marshal returns: @@ -110,7 +110,7 @@ func (c codec) Marshal(value interface{}) ([]byte, error) { // and returns the number of bytes it wrote. // When these functions are called in order, they write [value] to a byte slice. // 3) An error -func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err error) { +func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) error, err error) { valueKind := value.Kind() // Case: Value can't be marshalled @@ -125,116 +125,73 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err switch valueKind { case reflect.Uint8: size = 1 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 1 { - return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) - } - copy(b, []byte{byte(value.Uint())}) - return nil + f = func(p *wrappers.Packer) error { + p.PackByte(byte(value.Uint())) + return p.Err } return case reflect.Int8: size = 1 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 1 { - return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) - } - copy(b, []byte{byte(value.Int())}) - return nil + f = func(p *wrappers.Packer) error { + p.PackByte(byte(value.Int())) + return p.Err } return case reflect.Uint16: size = 2 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 2 { - return fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) - } - binary.BigEndian.PutUint16(b, uint16(value.Uint())) - return nil + f = func(p *wrappers.Packer) error { + p.PackShort(uint16(value.Uint())) + return p.Err } return case reflect.Int16: size = 2 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 2 { - return fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) - } - binary.BigEndian.PutUint16(b, uint16(value.Int())) - return nil + f = func(p *wrappers.Packer) error { + p.PackShort(uint16(value.Int())) + return p.Err } return case reflect.Uint32: size = 4 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 4 { - return fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) - } - binary.BigEndian.PutUint32(b, uint32(value.Uint())) - return nil + f = func(p *wrappers.Packer) error { + p.PackInt(uint32(value.Uint())) + return p.Err } return case reflect.Int32: size = 4 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 4 { - return fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) - } - binary.BigEndian.PutUint32(b, uint32(value.Int())) - return nil + f = func(p *wrappers.Packer) error { + p.PackInt(uint32(value.Int())) + return p.Err } return case reflect.Uint64: size = 8 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 8 { - return fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) - } - binary.BigEndian.PutUint64(b, uint64(value.Uint())) - return nil + f = func(p *wrappers.Packer) error { + p.PackLong(uint64(value.Uint())) + return p.Err } return case reflect.Int64: size = 8 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 8 { - return fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) - } - binary.BigEndian.PutUint64(b, uint64(value.Int())) - return nil + f = func(p *wrappers.Packer) error { + p.PackLong(uint64(value.Int())) + return p.Err } return case reflect.String: asStr := value.String() - strSize := len(asStr) - if strSize > maxStringLen { - return 0, nil, errSliceTooLarge - } - - size = strSize + 2 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < size { - return fmt.Errorf("expected len(bytes) to be at least %d but is %d", size, bytesLen) - } - binary.BigEndian.PutUint16(b, uint16(strSize)) - if strSize == 0 { - return nil - } - copy(b[2:], []byte(asStr)) - return nil + size = len(asStr) + 2 + f = func(p *wrappers.Packer) error { + p.PackStr(asStr) + return p.Err } return case reflect.Bool: size = 1 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 1 { - return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) - } - if value.Bool() { - copy(b, []byte{1}) - } else { - copy(b, []byte{0}) - } - return nil + f = func(p *wrappers.Packer) error { + p.PackBool(value.Bool()) + return p.Err } return case reflect.Uintptr, reflect.Ptr: @@ -251,51 +208,41 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err } size = 4 + subsize // 4 because we pack the type ID, a uint32 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 4+subsize { - return fmt.Errorf("expected len(bytes) to be at least %d but is %d", 4+subsize, bytesLen) + f = func(p *wrappers.Packer) error { + p.PackInt(typeID) + if p.Err != nil { + return p.Err } - binary.BigEndian.PutUint32(b, uint32(typeID)) - if len(b) == 4 { - return nil - } - return subfunc(b[4:]) + return subfunc(p) } return case reflect.Slice: if value.IsNil() { size = 1 - f = func(b []byte) error { - if bytesLen := len(b); bytesLen < 1 { - return fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) - } - b[0] = 1 // slice is nil; set isNil flag to 1 - return nil + f = func(p *wrappers.Packer) error { + p.PackBool(true) // slice is nil; set isNil flag to 1 + return p.Err } return } - numElts := value.Len() // # elements in the slice/array (assumed to be <= 2^31 - 1) + numElts := value.Len() // # elements in the slice/array (assumed to be <= math.MaxUint16) if numElts > c.maxSliceLen { return 0, nil, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) } - size = 5 // 1 for the isNil flag. 0 --> this slice isn't nil. 1--> it is nil. - // 4 for the size of the slice (uint32) + size = 3 // 1 for the isNil flag. 2 for the size of the slice (uint16) // offsets[i] is the index in the byte array that subFuncs[i] will start writing at offsets := make([]int, numElts+1, numElts+1) if numElts != 0 { - offsets[1] = 5 // 1 for nil flag, 4 for slice size + offsets[1] = 3 } - subFuncs := make([]func([]byte) error, numElts+1, numElts+1) - subFuncs[0] = func(b []byte) error { // write the nil flag and number of elements - if bytesLen := len(b); bytesLen < 5 { - return fmt.Errorf("expected len(bytes) to be at least 5 but is %d", bytesLen) - } - b[0] = 0 // slice is non-nil; set isNil flag to 0 - binary.BigEndian.PutUint32(b[1:], uint32(numElts)) - return nil + subFuncs := make([]func(*wrappers.Packer) error, numElts+1, numElts+1) + subFuncs[0] = func(p *wrappers.Packer) error { // write the nil flag and number of elements + p.PackBool(false) + p.PackShort(uint16(numElts)) + return p.Err } for i := 1; i < numElts+1; i++ { // Process each element in the slice subSize, subFunc, subErr := c.marshal(value.Index(i - 1)) @@ -313,14 +260,9 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) } - f = func(b []byte) error { - bytesLen := len(b) - for i, f := range subFuncs { - offset := offsets[i] - if offset > bytesLen { - return fmt.Errorf("attempted out of bounds slice. offset: %d. bytesLen: %d", offset, bytesLen) - } - if err := f(b[offset:]); err != nil { + f = func(p *wrappers.Packer) error { + for _, f := range subFuncs { + if err := f(p); err != nil { return err } } @@ -328,7 +270,7 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err } return case reflect.Array: - numElts := value.Len() // # elements in the slice/array (assumed to be <= 2^31 - 1) + numElts := value.Len() // # elements in the slice/array (assumed to be <= math.MaxUint16) if numElts > math.MaxUint32 { return 0, nil, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) } @@ -337,7 +279,7 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err // offsets[i] is the index in the byte array that subFuncs[i] will start writing at offsets := make([]int, numElts, numElts) offsets[1] = 4 // 4 for slice size - subFuncs := make([]func([]byte) error, numElts, numElts) + subFuncs := make([]func(*wrappers.Packer) error, numElts, numElts) for i := 0; i < numElts; i++ { // Process each element in the array subSize, subFunc, subErr := c.marshal(value.Index(i)) if subErr != nil { @@ -354,14 +296,9 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) } - f = func(b []byte) error { - bytesLen := len(b) - for i, f := range subFuncs { - offset := offsets[i] - if offset > bytesLen { - return fmt.Errorf("attempted out of bounds slice") - } - if err := f(b[offset:]); err != nil { + f = func(p *wrappers.Packer) error { + for _, f := range subFuncs { + if err := f(p); err != nil { return err } } @@ -375,7 +312,7 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err // offsets[i] is the index in the byte array that subFuncs[i] will start writing at offsets := make([]int, 0, numFields) offsets = append(offsets, 0) - subFuncs := make([]func([]byte) error, 0, numFields) + subFuncs := make([]func(*wrappers.Packer) error, 0, numFields) for i := 0; i < numFields; i++ { // Go through all fields of this struct field := t.Field(i) if !shouldSerialize(field) { // Skip fields we don't need to serialize @@ -396,14 +333,9 @@ func (c codec) marshal(value reflect.Value) (size int, f func([]byte) error, err } } - f = func(b []byte) error { - bytesLen := len(b) - for i, f := range subFuncs { - offset := offsets[i] - if offset > bytesLen { - return fmt.Errorf("attempted out of bounds slice") - } - if err := f(b[offset:]); err != nil { + f = func(p *wrappers.Packer) error { + for _, f := range subFuncs { + if err := f(p); err != nil { return err } } @@ -430,94 +362,95 @@ func (c codec) Unmarshal(bytes []byte, dest interface{}) error { return errNeedPointer } + p := &wrappers.Packer{MaxSize: c.maxSize, Bytes: bytes} destVal := destPtr.Elem() - bytesRead, err := c.unmarshal(bytes, destVal) - if err != nil { + if err := c.unmarshal(p, destVal); err != nil { return err } - if l := len(bytes); l != bytesRead { - return fmt.Errorf("%d leftover bytes after unmarshalling", l-bytesRead) - } return nil } // Unmarshal bytes from [bytes] into [field] // [field] must be addressable -// Returns the number of bytes read from [bytes] -func (c codec) unmarshal(bytes []byte, field reflect.Value) (int, error) { - bytesLen := len(bytes) +func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { kind := field.Kind() switch kind { case reflect.Uint8: - if bytesLen < 1 { - return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + b := p.UnpackByte() + if p.Err != nil { + return p.Err } - field.SetUint(uint64(bytes[0])) - return 1, nil + field.SetUint(uint64(b)) + return nil case reflect.Int8: - if bytesLen < 1 { - return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + b := p.UnpackByte() + if p.Err != nil { + return p.Err } - field.SetInt(int64(bytes[0])) - return 1, nil + field.SetInt(int64(b)) + return nil case reflect.Uint16: - if bytesLen < 2 { - return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + b := p.UnpackShort() + if p.Err != nil { + return p.Err } - field.SetUint(uint64(binary.BigEndian.Uint16(bytes))) - return 2, nil + field.SetUint(uint64(b)) + return nil case reflect.Int16: - if bytesLen < 2 { - return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + b := p.UnpackShort() + if p.Err != nil { + return p.Err } - field.SetInt(int64(binary.BigEndian.Uint16(bytes))) - return 2, nil + field.SetInt(int64(b)) + return nil case reflect.Uint32: - if bytesLen < 4 { - return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + b := p.UnpackInt() + if p.Err != nil { + return p.Err } - field.SetUint(uint64(binary.BigEndian.Uint32(bytes))) - return 4, nil + field.SetUint(uint64(b)) + return nil case reflect.Int32: - if bytesLen < 4 { - return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + b := p.UnpackInt() + if p.Err != nil { + return p.Err } - field.SetInt(int64(binary.BigEndian.Uint32(bytes))) - return 4, nil + field.SetInt(int64(b)) + return nil case reflect.Uint64: - if bytesLen < 4 { - return 0, fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + b := p.UnpackLong() + if p.Err != nil { + return p.Err } - field.SetUint(uint64(binary.BigEndian.Uint64(bytes))) - return 8, nil + field.SetUint(uint64(b)) + return nil case reflect.Int64: - if bytesLen < 4 { - return 0, fmt.Errorf("expected len(bytes) to be at least 8 but is %d", bytesLen) + b := p.UnpackLong() + if p.Err != nil { + return p.Err } - field.SetInt(int64(binary.BigEndian.Uint64(bytes))) - return 8, nil + field.SetInt(int64(b)) + return nil case reflect.Bool: - if bytesLen < 1 { - return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + b := p.UnpackBool() + if p.Err != nil { + return p.Err } - if bytes[0] == 0 { - field.SetBool(false) - } else { - field.SetBool(true) - } - return 1, nil + field.SetBool(b) + return nil case reflect.Slice: - if bytesLen < 1 { - return 0, fmt.Errorf("expected len(bytes) to be at least 1 but is %d", bytesLen) + isNil := p.UnpackBool() + if p.Err != nil { + return p.Err } - if bytes[0] == 1 { // isNil flag is 1 --> this slice is nil - return 1, nil + if isNil { // slice is nil + return nil } - numElts := int(binary.BigEndian.Uint32(bytes[1:])) // number of elements in the slice - if numElts > c.maxSliceLen { - return 0, fmt.Errorf("slice length, %d, exceeds maximum, %d", numElts, c.maxSliceLen) + numElts := int(p.UnpackShort()) + if p.Err != nil { + return p.Err } // set [field] to be a slice of the appropriate type/capacity (right now [field] is nil) @@ -525,113 +458,83 @@ func (c codec) unmarshal(bytes []byte, field reflect.Value) (int, error) { field.Set(slice) // Unmarshal each element into the appropriate index of the slice - bytesRead := 5 // 1 for isNil flag, 4 for numElts for i := 0; i < numElts; i++ { - if bytesRead > bytesLen { - return 0, fmt.Errorf("attempted out of bounds slice") + if err := c.unmarshal(p, field.Index(i)); err != nil { + return err } - n, err := c.unmarshal(bytes[bytesRead:], field.Index(i)) - if err != nil { - return 0, err - } - bytesRead += n } - return bytesRead, nil + return nil case reflect.Array: - bytesRead := 0 for i := 0; i < field.Len(); i++ { - if bytesRead > bytesLen { - return 0, fmt.Errorf("attempted out of bounds slice") + if err := c.unmarshal(p, field.Index(i)); err != nil { + return err } - n, err := c.unmarshal(bytes[bytesRead:], field.Index(i)) - if err != nil { - return 0, err - } - bytesRead += n } - return bytesRead, nil + return nil case reflect.String: - if bytesLen < 2 { - return 0, fmt.Errorf("expected len(bytes) to be at least 2 but is %d", bytesLen) + str := p.UnpackStr() + if p.Err != nil { + return p.Err } - strLen := int(binary.BigEndian.Uint16(bytes)) - if bytesLen < 2+strLen { - return 0, fmt.Errorf("expected len(bytes) to be at least %d but is %d", 2+strLen, bytesLen) - } - if strLen > 0 { - field.SetString(string(bytes[2 : 2+strLen])) - } else { - field.SetString("") - } - return strLen + 2, nil + field.SetString(str) + return nil case reflect.Interface: - if bytesLen < 4 { - return 0, fmt.Errorf("expected len(bytes) to be at least 4 but is %d", bytesLen) + typeID := p.UnpackInt() // Get the type ID + if p.Err != nil { + return p.Err } - - // Get the type ID - typeID := binary.BigEndian.Uint32(bytes) // Get a struct that implements the interface typ, ok := c.typeIDToType[typeID] if !ok { - return 0, errUnmarshalUnregisteredType + return errUnmarshalUnregisteredType } // Ensure struct actually does implement the interface fieldType := field.Type() if !typ.Implements(fieldType) { - return 0, fmt.Errorf("%s does not implement interface %s", typ, fieldType) + return fmt.Errorf("%s does not implement interface %s", typ, fieldType) } concreteInstancePtr := reflect.New(typ) // instance of the proper type // Unmarshal into the struct - - n, err := c.unmarshal(bytes[4:], concreteInstancePtr.Elem()) - if err != nil { - return 0, err + if err := c.unmarshal(p, concreteInstancePtr.Elem()); err != nil { + return err } // And assign the filled struct to the field field.Set(concreteInstancePtr.Elem()) - return n + 4, nil + return nil case reflect.Struct: // Type of this struct structType := reflect.TypeOf(field.Interface()) // Go through all the fields and umarshal into each - bytesRead := 0 for i := 0; i < structType.NumField(); i++ { structField := structType.Field(i) if !shouldSerialize(structField) { // Skip fields we don't need to unmarshal continue } if unicode.IsLower(rune(structField.Name[0])) { // Only unmarshal into exported field - return 0, errUnmarshalUnexportedField + return errUnmarshalUnexportedField } - field := field.Field(i) // Get the field - if bytesRead > bytesLen { - return 0, fmt.Errorf("attempted out of bounds slice") + field := field.Field(i) // Get the field + if err := c.unmarshal(p, field); err != nil { // Unmarshal into the field + return err } - n, err := c.unmarshal(bytes[bytesRead:], field) // Unmarshal into the field - if err != nil { - return 0, err - } - bytesRead += n } - return bytesRead, nil + return nil case reflect.Ptr: // Get the type this pointer points to underlyingType := field.Type().Elem() // Create a new pointer to a new value of the underlying type underlyingValue := reflect.New(underlyingType) // Fill the value - n, err := c.unmarshal(bytes, underlyingValue.Elem()) - if err != nil { - return 0, err + if err := c.unmarshal(p, underlyingValue.Elem()); err != nil { + return err } // Assign to the top-level struct's member field.Set(underlyingValue) - return n, nil + return nil case reflect.Invalid: - return 0, errNil + return errNil default: - return 0, errUnknownType + return errUnknownType } } diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index 7100a79..8bf22a1 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -474,7 +474,7 @@ func TestEmptySliceSerialization(t *testing.T) { codec := NewDefault() val := &simpleSliceStruct{Arr: make([]uint32, 0, 1)} - expected := []byte{0, 0, 0, 0, 0} // 0 for isNil flag, 0 for size + expected := []byte{0, 0, 0} // 0 for isNil flag, 0 for size result, err := codec.Marshal(val) if err != nil { t.Fatal(err) @@ -507,7 +507,7 @@ func TestSliceWithEmptySerialization(t *testing.T) { val := &nestedSliceStruct{ Arr: make([]emptyStruct, 1000), } - expected := []byte{0x00, 0x00, 0x00, 0x03, 0xE8} // 0 for isNil flag, then 1000 for numElts + expected := []byte{0x00, 0x03, 0xE8} // 0 for isNil flag, then 1000 for numElts result, err := codec.Marshal(val) if err != nil { t.Fatal(err) From e15c1bad8cef1d6496497a20177df4a4b263a3cb Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Tue, 9 Jun 2020 13:51:11 -0400 Subject: [PATCH 11/42] Use Add instead of Push to correctly order timed txs in event heap --- vms/platformvm/service.go | 2 +- vms/platformvm/static_service.go | 7 ++- vms/platformvm/static_service_test.go | 69 +++++++++++++++++++++++++++ vms/platformvm/vm.go | 7 ++- vms/platformvm/vm_test.go | 5 +- 5 files changed, 78 insertions(+), 12 deletions(-) diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index de2d41b..05723dd 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -1275,7 +1275,7 @@ func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *Is if err := tx.initialize(service.vm); err != nil { return fmt.Errorf("error initializing tx: %s", err) } - service.vm.unissuedEvents.Push(tx) + service.vm.unissuedEvents.Add(tx) response.TxID = tx.ID() case DecisionTx: if err := tx.initialize(service.vm); err != nil { diff --git a/vms/platformvm/static_service.go b/vms/platformvm/static_service.go index 8acc0a9..1cdeeca 100644 --- a/vms/platformvm/static_service.go +++ b/vms/platformvm/static_service.go @@ -4,7 +4,6 @@ package platformvm import ( - "container/heap" "errors" "net/http" @@ -174,8 +173,8 @@ func (*StaticService) BuildGenesis(_ *http.Request, args *BuildGenesisArgs, repl return errAccountHasNoValue } accounts = append(accounts, newAccount( - account.Address, // ID - 0, // nonce + account.Address, // ID + 0, // nonce uint64(account.Balance), // balance )) } @@ -210,7 +209,7 @@ func (*StaticService) BuildGenesis(_ *http.Request, args *BuildGenesisArgs, repl return err } - heap.Push(validators, tx) + validators.Add(tx) } // Specify the chains that exist at genesis. diff --git a/vms/platformvm/static_service_test.go b/vms/platformvm/static_service_test.go index 04433ff..b0cb494 100644 --- a/vms/platformvm/static_service_test.go +++ b/vms/platformvm/static_service_test.go @@ -111,3 +111,72 @@ func TestBuildGenesisInvalidEndtime(t *testing.T) { t.Fatalf("Should have errored due to an invalid end time") } } + +func TestBuildGenesisReturnsSortedValidators(t *testing.T) { + id := ids.NewShortID([20]byte{1}) + account := APIAccount{ + Address: id, + Balance: 123456789, + } + + weight := json.Uint64(987654321) + validator1 := APIDefaultSubnetValidator{ + APIValidator: APIValidator{ + StartTime: 0, + EndTime: 20, + Weight: &weight, + ID: id, + }, + Destination: id, + } + + validator2 := APIDefaultSubnetValidator{ + APIValidator: APIValidator{ + StartTime: 3, + EndTime: 15, + Weight: &weight, + ID: id, + }, + Destination: id, + } + + validator3 := APIDefaultSubnetValidator{ + APIValidator: APIValidator{ + StartTime: 1, + EndTime: 10, + Weight: &weight, + ID: id, + }, + Destination: id, + } + + args := BuildGenesisArgs{ + Accounts: []APIAccount{ + account, + }, + Validators: []APIDefaultSubnetValidator{ + validator1, + validator2, + validator3, + }, + Time: 5, + } + reply := BuildGenesisReply{} + + ss := StaticService{} + if err := ss.BuildGenesis(nil, &args, &reply); err != nil { + t.Fatalf("BuildGenesis should not have errored") + } + + genesis := &Genesis{} + Codec.Unmarshal(reply.Bytes.Bytes, genesis) + validators := genesis.Validators + currentValidator := validators.Remove() + for validators.Len() > 0 { + nextValidator := validators.Remove() + if currentValidator.EndTime().Unix() > nextValidator.EndTime().Unix() { + t.Fatalf("Validators returned by genesis should be a min heap sorted by end time") + } + currentValidator = nextValidator + } +} diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index 9f1ce53..8d5a71c 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -4,7 +4,6 @@ package platformvm import ( - "container/heap" "errors" "fmt" "time" @@ -698,7 +697,7 @@ func (vm *VM) resetTimer() { vm.SnowmanVM.NotifyBlockReady() // Should issue a ProposeAddValidator return } - // If the tx doesn't meet the syncrony bound, drop it + // If the tx doesn't meet the synchrony bound, drop it vm.unissuedEvents.Remove() vm.Ctx.Log.Debug("dropping tx to add validator because its start time has passed") } @@ -780,8 +779,8 @@ func (vm *VM) calculateValidators(db database.Database, timestamp time.Time, sub if timestamp.Before(nextTx.StartTime()) { break } - heap.Push(current, nextTx) - heap.Pop(pending) + current.Add(nextTx) + pending.Remove() started.Add(nextTx.Vdr().ID()) } return current, pending, started, stopped, nil diff --git a/vms/platformvm/vm_test.go b/vms/platformvm/vm_test.go index b8bb47c..182c44a 100644 --- a/vms/platformvm/vm_test.go +++ b/vms/platformvm/vm_test.go @@ -5,7 +5,6 @@ package platformvm import ( "bytes" - "container/heap" "errors" "testing" "time" @@ -226,7 +225,7 @@ func GenesisCurrentValidators() *EventHeap { testNetworkID, // network ID key, // key paying tx fee and stake ) - heap.Push(validators, validator) + validators.Add(validator) } return validators } @@ -1011,7 +1010,7 @@ func TestCreateSubnet(t *testing.T) { t.Fatal(err) } - vm.unissuedEvents.Push(addValidatorTx) + vm.unissuedEvents.Add(addValidatorTx) blk, err = vm.BuildBlock() // should add validator to the new subnet if err != nil { t.Fatal(err) From 26f5503a432e5eb02b803c70c5de2f2beb25d362 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Wed, 10 Jun 2020 22:10:19 -0400 Subject: [PATCH 12/42] Add test to ensure IssueTx maintains ordering of unissued events heap --- vms/platformvm/service.go | 2 +- vms/platformvm/service_test.go | 184 ++++++++++++++++++++++++++ vms/platformvm/static_service_test.go | 7 +- vms/platformvm/vm_test.go | 2 + 4 files changed, 193 insertions(+), 2 deletions(-) diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 05723dd..78316c5 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -1290,7 +1290,7 @@ func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *Is service.vm.unissuedAtomicTxs = append(service.vm.unissuedAtomicTxs, tx) response.TxID = tx.ID() default: - return errors.New("Could not parse given tx. Must be a TimedTx, DecisionTx, or AtomicTx") + return errors.New("Could not parse given tx. Must not be a TimedTx, DecisionTx, or AtomicTx") } service.vm.resetTimer() diff --git a/vms/platformvm/service_test.go b/vms/platformvm/service_test.go index 9ac4a6c..b6e4a31 100644 --- a/vms/platformvm/service_test.go +++ b/vms/platformvm/service_test.go @@ -6,6 +6,9 @@ package platformvm import ( "encoding/json" "testing" + "time" + + "github.com/ava-labs/gecko/utils/formatting" ) func TestAddDefaultSubnetValidator(t *testing.T) { @@ -50,3 +53,184 @@ func TestImportKey(t *testing.T) { t.Fatal(err) } } + +func TestIssueTxKeepsTimedEventsSorted(t *testing.T) { + vm := defaultVM() + vm.Ctx.Lock.Lock() + defer func() { + vm.Shutdown() + vm.Ctx.Lock.Unlock() + }() + + service := Service{vm: vm} + + pendingValidatorStartTime1 := defaultGenesisTime.Add(3 * time.Second) + pendingValidatorEndTime1 := pendingValidatorStartTime1.Add(MinimumStakingDuration) + nodeIDKey1, _ := vm.factory.NewPrivateKey() + nodeID1 := nodeIDKey1.PublicKey().Address() + addPendingValidatorTx1, err := vm.newAddDefaultSubnetValidatorTx( + defaultNonce+1, + defaultStakeAmount, + uint64(pendingValidatorStartTime1.Unix()), + uint64(pendingValidatorEndTime1.Unix()), + nodeID1, + nodeID1, + NumberOfShares, + testNetworkID, + defaultKey, + ) + if err != nil { + t.Fatal(err) + } + + txBytes1, err := Codec.Marshal(genericTx{Tx: addPendingValidatorTx1}) + if err != nil { + t.Fatal(err) + } + + args1 := &IssueTxArgs{} + args1.Tx = formatting.CB58{Bytes: txBytes1} + reply1 := IssueTxResponse{} + + err = service.IssueTx(nil, args1, &reply1) + if err != nil { + t.Fatal(err) + } + + pendingValidatorStartTime2 := defaultGenesisTime.Add(2 * time.Second) + pendingValidatorEndTime2 := pendingValidatorStartTime2.Add(MinimumStakingDuration) + nodeIDKey2, _ := vm.factory.NewPrivateKey() + nodeID2 := nodeIDKey2.PublicKey().Address() + addPendingValidatorTx2, err := vm.newAddDefaultSubnetValidatorTx( + defaultNonce+1, + defaultStakeAmount, + uint64(pendingValidatorStartTime2.Unix()), + uint64(pendingValidatorEndTime2.Unix()), + nodeID2, + nodeID2, + NumberOfShares, + testNetworkID, + defaultKey, + ) + if err != nil { + t.Fatal(err) + } + + txBytes2, err := Codec.Marshal(genericTx{Tx: addPendingValidatorTx2}) + if err != nil { + t.Fatal(err) + } + + args2 := IssueTxArgs{Tx: formatting.CB58{Bytes: txBytes2}} + reply2 := IssueTxResponse{} + + err = service.IssueTx(nil, &args2, &reply2) + if err != nil { + t.Fatal(err) + } + + pendingValidatorStartTime3 := defaultGenesisTime.Add(10 * time.Second) + pendingValidatorEndTime3 := pendingValidatorStartTime3.Add(MinimumStakingDuration) + nodeIDKey3, _ := vm.factory.NewPrivateKey() + nodeID3 := nodeIDKey3.PublicKey().Address() + addPendingValidatorTx3, err := vm.newAddDefaultSubnetValidatorTx( + defaultNonce+1, + defaultStakeAmount, + uint64(pendingValidatorStartTime3.Unix()), + uint64(pendingValidatorEndTime3.Unix()), + nodeID3, + nodeID3, + NumberOfShares, + testNetworkID, + defaultKey, + ) + if err != nil { + t.Fatal(err) + } + + txBytes3, err := Codec.Marshal(genericTx{Tx: addPendingValidatorTx3}) + if err != nil { + t.Fatal(err) + } + + args3 := IssueTxArgs{Tx: formatting.CB58{Bytes: txBytes3}} + reply3 := IssueTxResponse{} + + err = service.IssueTx(nil, &args3, &reply3) + if err != nil { + t.Fatal(err) + } + + pendingValidatorStartTime4 := defaultGenesisTime.Add(1 * time.Second) + pendingValidatorEndTime4 := pendingValidatorStartTime4.Add(MinimumStakingDuration) + nodeIDKey4, _ := vm.factory.NewPrivateKey() + nodeID4 := nodeIDKey4.PublicKey().Address() + addPendingValidatorTx4, err := vm.newAddDefaultSubnetValidatorTx( + defaultNonce+1, + defaultStakeAmount, + uint64(pendingValidatorStartTime4.Unix()), + uint64(pendingValidatorEndTime4.Unix()), + nodeID4, + nodeID4, + NumberOfShares, + testNetworkID, + defaultKey, + ) + if err != nil { + t.Fatal(err) + } + + txBytes4, err := Codec.Marshal(genericTx{Tx: addPendingValidatorTx4}) + if err != nil { + t.Fatal(err) + } + + args4 := IssueTxArgs{Tx: formatting.CB58{Bytes: txBytes4}} + reply4 := IssueTxResponse{} + + err = service.IssueTx(nil, &args4, &reply4) + if err != nil { + t.Fatal(err) + } + + pendingValidatorStartTime5 := defaultGenesisTime.Add(50 * time.Second) + pendingValidatorEndTime5 := pendingValidatorStartTime5.Add(MinimumStakingDuration) + nodeIDKey5, _ := vm.factory.NewPrivateKey() + nodeID5 := nodeIDKey5.PublicKey().Address() + addPendingValidatorTx5, err := vm.newAddDefaultSubnetValidatorTx( + defaultNonce+1, + defaultStakeAmount, + uint64(pendingValidatorStartTime5.Unix()), + uint64(pendingValidatorEndTime5.Unix()), + nodeID5, + nodeID5, + NumberOfShares, + testNetworkID, + defaultKey, + ) + if err != nil { + t.Fatal(err) + } + + txBytes5, err := Codec.Marshal(genericTx{Tx: addPendingValidatorTx5}) + if err != nil { + t.Fatal(err) + } + + args5 := IssueTxArgs{Tx: formatting.CB58{Bytes: txBytes5}} + reply5 := IssueTxResponse{} + + err = service.IssueTx(nil, &args5, &reply5) + if err != nil { + t.Fatal(err) + } + + currentEvent := vm.unissuedEvents.Remove() + for vm.unissuedEvents.Len() > 0 { + nextEvent := vm.unissuedEvents.Remove() + if !currentEvent.StartTime().Before(nextEvent.StartTime()) { + t.Fatal("IssueTx does not keep event heap ordered") + } + currentEvent = nextEvent + } +} diff --git a/vms/platformvm/static_service_test.go b/vms/platformvm/static_service_test.go index b0cb494..3f64a9b 100644 --- a/vms/platformvm/static_service_test.go +++ b/vms/platformvm/static_service_test.go @@ -169,8 +169,13 @@ func TestBuildGenesisReturnsSortedValidators(t *testing.T) { } genesis := &Genesis{} - Codec.Unmarshal(reply.Bytes.Bytes, genesis) + if err := Codec.Unmarshal(reply.Bytes.Bytes, genesis); err != nil { + t.Fatal(err) + } validators := genesis.Validators + if validators.Len() == 0 { + t.Fatal("Validators should contain 3 validators") + } currentValidator := validators.Remove() for validators.Len() > 0 { nextValidator := validators.Remove() diff --git a/vms/platformvm/vm_test.go b/vms/platformvm/vm_test.go index 182c44a..dcee89a 100644 --- a/vms/platformvm/vm_test.go +++ b/vms/platformvm/vm_test.go @@ -192,6 +192,8 @@ func defaultVM() *VM { panic("no subnets found") } // end delete + vm.registerDBTypes() + return vm } From 750f7b212074879e941d4049478dd22b6759f612 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 12 Jun 2020 13:48:45 -0400 Subject: [PATCH 13/42] Improve error message in platform API IssueTx call --- vms/platformvm/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 78316c5..91d18c4 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -1290,7 +1290,7 @@ func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *Is service.vm.unissuedAtomicTxs = append(service.vm.unissuedAtomicTxs, tx) response.TxID = tx.ID() default: - return errors.New("Could not parse given tx. Must not be a TimedTx, DecisionTx, or AtomicTx") + return errors.New("Could not parse given tx. Provided tx needs to be a TimedTx, DecisionTx, or AtomicTx") } service.vm.resetTimer() From 1572b1bd97ad11be54837b2e3fcfecf3bace9bfe Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 12 Jun 2020 14:09:45 -0400 Subject: [PATCH 14/42] Return early when finding address already exists --- vms/avm/service.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/vms/avm/service.go b/vms/avm/service.go index 1dce66d..56e5599 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -670,18 +670,16 @@ func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *I return fmt.Errorf("problem saving key while getting existing addresses: %w", err) } newAddress := sk.PublicKey().Address() - exists := false for _, address := range addresses { if newAddress.Equals(address) { - exists = true + reply.Address = service.vm.Format(newAddress.Bytes()) + return nil } } - if !exists { - addresses = append(addresses, newAddress) - if err := user.SetAddresses(db, addresses); err != nil { - return fmt.Errorf("problem saving addresses: %w", err) - } + addresses = append(addresses, newAddress) + if err := user.SetAddresses(db, addresses); err != nil { + return fmt.Errorf("problem saving addresses: %w", err) } reply.Address = service.vm.Format(newAddress.Bytes()) From 2fb88906cc15ca8734fb520202e774141f41bdfc Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 12 Jun 2020 15:10:39 -0400 Subject: [PATCH 15/42] Add testing for avm API ImportKey --- api/keystore/service.go | 70 +++++++++++++--------- api/keystore/service_test.go | 34 ++++------- vms/avm/service.go | 6 +- vms/avm/service_test.go | 112 +++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 54 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 16aca06..5c0d4e4 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -8,12 +8,14 @@ import ( "fmt" "net/http" "sync" + "testing" "github.com/gorilla/rpc/v2" "github.com/ava-labs/gecko/chains/atomic" "github.com/ava-labs/gecko/database" "github.com/ava-labs/gecko/database/encdb" + "github.com/ava-labs/gecko/database/memdb" "github.com/ava-labs/gecko/database/prefixdb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow/engine/common" @@ -137,35 +139,9 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre ks.log.Verbo("CreateUser called with %.*s", maxUserPassLen, args.Username) - if len(args.Username) > maxUserPassLen || len(args.Password) > maxUserPassLen { - return errUserPassMaxLength - } - - if args.Username == "" { - return errEmptyUsername - } - if usr, err := ks.getUser(args.Username); err == nil || usr != nil { - return fmt.Errorf("user already exists: %s", args.Username) - } - - if zxcvbn.PasswordStrength(args.Password, nil).Score < requiredPassScore { - return errWeakPassword - } - - usr := &User{} - if err := usr.Initialize(args.Password); err != nil { + if err := ks.AddUser(args.Username, args.Password); err != nil { return err } - - usrBytes, err := ks.codec.Marshal(usr) - if err != nil { - return err - } - - if err := ks.userDB.Put([]byte(args.Username), usrBytes); err != nil { - return err - } - ks.users[args.Username] = usr reply.Success = true return nil } @@ -403,3 +379,43 @@ func (ks *Keystore) GetDatabase(bID ids.ID, username, password string) (database return encDB, nil } + +func (ks *Keystore) AddUser(username, password string) error { + if len(username) > maxUserPassLen || len(password) > maxUserPassLen { + return errUserPassMaxLength + } + + if username == "" { + return errEmptyUsername + } + if usr, err := ks.getUser(username); err == nil || usr != nil { + return fmt.Errorf("user already exists: %s", username) + } + + if zxcvbn.PasswordStrength(password, nil).Score < requiredPassScore { + return errWeakPassword + } + + usr := &User{} + if err := usr.Initialize(password); err != nil { + return err + } + + usrBytes, err := ks.codec.Marshal(usr) + if err != nil { + return err + } + + if err := ks.userDB.Put([]byte(username), usrBytes); err != nil { + return err + } + ks.users[username] = usr + + return nil +} + +func CreateTestKeystore(t *testing.T) *Keystore { + ks := &Keystore{} + ks.Initialize(logging.NoLog{}, memdb.New()) + return ks +} diff --git a/api/keystore/service_test.go b/api/keystore/service_test.go index 9ec5cfa..3e0b18f 100644 --- a/api/keystore/service_test.go +++ b/api/keystore/service_test.go @@ -10,9 +10,7 @@ import ( "reflect" "testing" - "github.com/ava-labs/gecko/database/memdb" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/utils/logging" ) var ( @@ -22,8 +20,7 @@ var ( ) func TestServiceListNoUsers(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) reply := ListUsersReply{} if err := ks.ListUsers(nil, &ListUsersArgs{}, &reply); err != nil { @@ -35,8 +32,7 @@ func TestServiceListNoUsers(t *testing.T) { } func TestServiceCreateUser(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -75,8 +71,7 @@ func genStr(n int) string { // TestServiceCreateUserArgsChecks generates excessively long usernames or // passwords to assure the santity checks on string length are not exceeded func TestServiceCreateUserArgsCheck(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -117,8 +112,7 @@ func TestServiceCreateUserArgsCheck(t *testing.T) { // TestServiceCreateUserWeakPassword tests creating a new user with a weak // password to ensure the password strength check is working func TestServiceCreateUserWeakPassword(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -138,8 +132,7 @@ func TestServiceCreateUserWeakPassword(t *testing.T) { } func TestServiceCreateDuplicate(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -166,8 +159,7 @@ func TestServiceCreateDuplicate(t *testing.T) { } func TestServiceCreateUserNoName(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) reply := CreateUserReply{} if err := ks.CreateUser(nil, &CreateUserArgs{ @@ -178,8 +170,7 @@ func TestServiceCreateUserNoName(t *testing.T) { } func TestServiceUseBlockchainDB(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -218,8 +209,7 @@ func TestServiceUseBlockchainDB(t *testing.T) { } func TestServiceExportImport(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) { reply := CreateUserReply{} @@ -252,8 +242,7 @@ func TestServiceExportImport(t *testing.T) { t.Fatal(err) } - newKS := Keystore{} - newKS.Initialize(logging.NoLog{}, memdb.New()) + newKS := CreateTestKeystore(t) { reply := ImportUserReply{} @@ -358,11 +347,10 @@ func TestServiceDeleteUser(t *testing.T) { for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { - ks := Keystore{} - ks.Initialize(logging.NoLog{}, memdb.New()) + ks := CreateTestKeystore(t) if tt.setup != nil { - if err := tt.setup(&ks); err != nil { + if err := tt.setup(ks); err != nil { t.Fatalf("failed to create user setup in keystore: %v", err) } } diff --git a/vms/avm/service.go b/vms/avm/service.go index 56e5599..37a37a3 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -665,10 +665,8 @@ func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *I return fmt.Errorf("problem saving key %w", err) } - addresses, err := user.Addresses(db) - if err != nil { - return fmt.Errorf("problem saving key while getting existing addresses: %w", err) - } + addresses, _ := user.Addresses(db) + newAddress := sk.PublicKey().Address() for _, address := range addresses { if newAddress.Equals(address) { diff --git a/vms/avm/service_test.go b/vms/avm/service_test.go index fdd8053..6e1d387 100644 --- a/vms/avm/service_test.go +++ b/vms/avm/service_test.go @@ -9,8 +9,10 @@ import ( "github.com/stretchr/testify/assert" + "github.com/ava-labs/gecko/api/keystore" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow/choices" + "github.com/ava-labs/gecko/utils/crypto" "github.com/ava-labs/gecko/utils/formatting" ) @@ -340,3 +342,113 @@ func TestCreateVariableCapAsset(t *testing.T) { t.Fatalf("Wrong assetID returned from CreateFixedCapAsset %s", reply.AssetID) } } + +func TestImportAvmKey(t *testing.T) { + _, vm, s := setup(t) + defer func() { + vm.Shutdown() + ctx.Lock.Unlock() + }() + + userKeystore := keystore.CreateTestKeystore(t) + + username := "bobby" + password := "StrnasfqewiurPasswdn56d" + if err := userKeystore.AddUser(username, password); err != nil { + t.Fatal(err) + } + + vm.ctx.Keystore = userKeystore.NewBlockchainKeyStore(vm.ctx.ChainID) + _, err := vm.ctx.Keystore.GetDatabase(username, password) + if err != nil { + t.Fatal(err) + } + + factory := crypto.FactorySECP256K1R{} + skIntf, err := factory.NewPrivateKey() + if err != nil { + t.Fatalf("problem generating private key: %w", err) + } + sk := skIntf.(*crypto.PrivateKeySECP256K1R) + + args := ImportKeyArgs{ + Username: username, + Password: password, + PrivateKey: formatting.CB58{Bytes: sk.Bytes()}, + } + reply := ImportKeyReply{} + if err = s.ImportKey(nil, &args, &reply); err != nil { + t.Fatal(err) + } +} + +func TestImportAvmKeyNoDuplicates(t *testing.T) { + _, vm, s := setup(t) + defer func() { + vm.Shutdown() + ctx.Lock.Unlock() + }() + + userKeystore := keystore.CreateTestKeystore(t) + + username := "bobby" + password := "StrnasfqewiurPasswdn56d" + if err := userKeystore.AddUser(username, password); err != nil { + t.Fatal(err) + } + + vm.ctx.Keystore = userKeystore.NewBlockchainKeyStore(vm.ctx.ChainID) + _, err := vm.ctx.Keystore.GetDatabase(username, password) + if err != nil { + t.Fatal(err) + } + + factory := crypto.FactorySECP256K1R{} + skIntf, err := factory.NewPrivateKey() + if err != nil { + t.Fatalf("problem generating private key: %w", err) + } + sk := skIntf.(*crypto.PrivateKeySECP256K1R) + + args := ImportKeyArgs{ + Username: username, + Password: password, + PrivateKey: formatting.CB58{Bytes: sk.Bytes()}, + } + reply := ImportKeyReply{} + if err = s.ImportKey(nil, &args, &reply); err != nil { + t.Fatal(err) + } + + expectedAddress := vm.Format(sk.PublicKey().Address().Bytes()) + + if reply.Address != expectedAddress { + t.Fatalf("Reply address: %s did not match expected address: %s", reply.Address, expectedAddress) + } + + reply2 := ImportKeyReply{} + if err = s.ImportKey(nil, &args, &reply2); err != nil { + t.Fatal(err) + } + + if reply2.Address != expectedAddress { + t.Fatalf("Reply address: %s did not match expected address: %s", reply2.Address, expectedAddress) + } + + addrsArgs := ListAddressesArgs{ + Username: username, + Password: password, + } + addrsReply := ListAddressesResponse{} + if err := s.ListAddresses(nil, &addrsArgs, &addrsReply); err != nil { + t.Fatal(err) + } + + if len(addrsReply.Addresses) != 1 { + t.Fatal("Importing the same key twice created duplicate addresses") + } + + if addrsReply.Addresses[0] != expectedAddress { + t.Fatal("List addresses returned an incorrect address") + } +} From a895b691a2f7cfbcd57abc2f8f7e7f04e7a39e0f Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 12 Jun 2020 16:52:58 -0400 Subject: [PATCH 16/42] change wire format to be exact same as it was before --- vms/components/codec/codec.go | 81 +++++++----------------------- vms/components/codec/codec_test.go | 20 +++----- 2 files changed, 25 insertions(+), 76 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 873aac2..182d8d2 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -15,7 +15,7 @@ import ( const ( defaultMaxSize = 1 << 18 // default max size, in bytes, of something being marshalled by Marshal() - defaultMaxSliceLength = 1 << 18 // default max length of a slice being marshalled by Marshal() + defaultMaxSliceLength = 1 << 18 // default max length of a slice being marshalled by Marshal(). Should be <= math.MaxUint32. maxStringLen = math.MaxUint16 ) @@ -86,6 +86,7 @@ func (c codec) RegisterType(val interface{}) error { // 5) To marshal an interface, you must pass a pointer to the value // 6) To unmarshal an interface, you must call codec.RegisterType([instance of the type that fulfills the interface]). // 7) Serialized fields must be exported +// 8) nil slices are marshaled as empty slices // To marshal an interface, [value] must be a pointer to the interface func (c codec) Marshal(value interface{}) ([]byte, error) { @@ -217,50 +218,27 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) } return case reflect.Slice: - if value.IsNil() { - size = 1 - f = func(p *wrappers.Packer) error { - p.PackBool(true) // slice is nil; set isNil flag to 1 - return p.Err - } - return - } - - numElts := value.Len() // # elements in the slice/array (assumed to be <= math.MaxUint16) + numElts := value.Len() // # elements in the slice/array. 0 if this slice is nil. if numElts > c.maxSliceLen { - return 0, nil, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) + return 0, nil, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } - size = 3 // 1 for the isNil flag. 2 for the size of the slice (uint16) - - // offsets[i] is the index in the byte array that subFuncs[i] will start writing at - offsets := make([]int, numElts+1, numElts+1) - if numElts != 0 { - offsets[1] = 3 - } - subFuncs := make([]func(*wrappers.Packer) error, numElts+1, numElts+1) - subFuncs[0] = func(p *wrappers.Packer) error { // write the nil flag and number of elements - p.PackBool(false) - p.PackShort(uint16(numElts)) - return p.Err - } - for i := 1; i < numElts+1; i++ { // Process each element in the slice - subSize, subFunc, subErr := c.marshal(value.Index(i - 1)) + subFuncs := make([]func(*wrappers.Packer) error, numElts, numElts) + size = wrappers.IntLen // for # elements + for i := 0; i < numElts; i++ { // Process each element in the slice + subSize, subFunc, subErr := c.marshal(value.Index(i)) if subErr != nil { return 0, nil, subErr } size += subSize - if i != numElts { // set offest for next function unless this is last ieration - offsets[i+1] = offsets[i] + subSize - } subFuncs[i] = subFunc } - if subFuncsLen := len(subFuncs); subFuncsLen != len(offsets) { - return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) - } - f = func(p *wrappers.Packer) error { + p.PackInt(uint32(numElts)) // pack # elements + if p.Err != nil { + return p.Err + } for _, f := range subFuncs { if err := f(p); err != nil { return err @@ -270,15 +248,12 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) } return case reflect.Array: - numElts := value.Len() // # elements in the slice/array (assumed to be <= math.MaxUint16) - if numElts > math.MaxUint32 { - return 0, nil, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, math.MaxUint32) + numElts := value.Len() + if numElts > c.maxSliceLen { + return 0, nil, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } size = 0 - // offsets[i] is the index in the byte array that subFuncs[i] will start writing at - offsets := make([]int, numElts, numElts) - offsets[1] = 4 // 4 for slice size subFuncs := make([]func(*wrappers.Packer) error, numElts, numElts) for i := 0; i < numElts; i++ { // Process each element in the array subSize, subFunc, subErr := c.marshal(value.Index(i)) @@ -286,16 +261,9 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) return 0, nil, subErr } size += subSize - if i != numElts-1 { // set offest for next function unless this is last ieration - offsets[i+1] = offsets[i] + subSize - } subFuncs[i] = subFunc } - if subFuncsLen := len(subFuncs); subFuncsLen != len(offsets) { - return 0, nil, fmt.Errorf("expected len(subFuncs) = %d. len(offsets) = %d. Should be same", subFuncsLen, len(offsets)) - } - f = func(p *wrappers.Packer) error { for _, f := range subFuncs { if err := f(p); err != nil { @@ -308,10 +276,8 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) case reflect.Struct: t := value.Type() numFields := t.NumField() + size = 0 - // offsets[i] is the index in the byte array that subFuncs[i] will start writing at - offsets := make([]int, 0, numFields) - offsets = append(offsets, 0) subFuncs := make([]func(*wrappers.Packer) error, 0, numFields) for i := 0; i < numFields; i++ { // Go through all fields of this struct field := t.Field(i) @@ -328,9 +294,6 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) } size += subSize subFuncs = append(subFuncs, subfunc) - if i != numFields-1 { // set offset for next function if not last iteration - offsets = append(offsets, offsets[len(offsets)-1]+subSize) - } } f = func(p *wrappers.Packer) error { @@ -440,23 +403,13 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { field.SetBool(b) return nil case reflect.Slice: - isNil := p.UnpackBool() + numElts := int(p.UnpackInt()) if p.Err != nil { return p.Err } - if isNil { // slice is nil - return nil - } - - numElts := int(p.UnpackShort()) - if p.Err != nil { - return p.Err - } - // set [field] to be a slice of the appropriate type/capacity (right now [field] is nil) slice := reflect.MakeSlice(field.Type(), numElts, numElts) field.Set(slice) - // Unmarshal each element into the appropriate index of the slice for i := 0; i < numElts; i++ { if err := c.unmarshal(p, field.Index(i)); err != nil { diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index 8bf22a1..f42e827 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -371,7 +371,7 @@ func TestString(t *testing.T) { } } -// Ensure a nil slice is marshaled/unmarshaled correctly +// Ensure a nil slice is unmarshaled to slice with length 0 func TestNilSlice(t *testing.T) { type structWithSlice struct { Slice []byte `serialize:"true"` @@ -389,8 +389,8 @@ func TestNilSlice(t *testing.T) { t.Fatal(err) } - if structUnmarshaled.Slice != nil { - t.Fatal("expected slice to be nil") + if structUnmarshaled.Slice == nil || len(structUnmarshaled.Slice) != 0 { + t.Fatal("expected slice to be non-nil and length 0") } } @@ -449,7 +449,7 @@ func TestNilSliceSerialization(t *testing.T) { codec := NewDefault() val := &simpleSliceStruct{} - expected := []byte{1} // 1 for isNil + expected := []byte{0, 0, 0, 0} // nil slice marshaled as 0 length slice result, err := codec.Marshal(val) if err != nil { t.Fatal(err) @@ -462,10 +462,8 @@ func TestNilSliceSerialization(t *testing.T) { valUnmarshaled := &simpleSliceStruct{} if err = codec.Unmarshal(result, &valUnmarshaled); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(valUnmarshaled, val) { - t.Logf("val: %v\n", val) - t.Logf("valUnmarshaled: %v\n", valUnmarshaled) - t.Fatal("should be same") + } else if len(valUnmarshaled.Arr) != 0 { + t.Fatal("should be 0 length") } } @@ -474,7 +472,7 @@ func TestEmptySliceSerialization(t *testing.T) { codec := NewDefault() val := &simpleSliceStruct{Arr: make([]uint32, 0, 1)} - expected := []byte{0, 0, 0} // 0 for isNil flag, 0 for size + expected := []byte{0, 0, 0, 0} // 0 for size result, err := codec.Marshal(val) if err != nil { t.Fatal(err) @@ -488,8 +486,6 @@ func TestEmptySliceSerialization(t *testing.T) { if err = codec.Unmarshal(result, &valUnmarshaled); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(valUnmarshaled, val) { - t.Logf("val: %v\n", val) - t.Logf("valUnmarshaled: %v\n", valUnmarshaled) t.Fatal("should be same") } } @@ -507,7 +503,7 @@ func TestSliceWithEmptySerialization(t *testing.T) { val := &nestedSliceStruct{ Arr: make([]emptyStruct, 1000), } - expected := []byte{0x00, 0x03, 0xE8} // 0 for isNil flag, then 1000 for numElts + expected := []byte{0x00, 0x00, 0x03, 0xE8} //1000 for numElts result, err := codec.Marshal(val) if err != nil { t.Fatal(err) From 42deac45e926dd5992973cfc89bc120d59a63ed8 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 12 Jun 2020 19:03:08 -0400 Subject: [PATCH 17/42] use pre-allocated array of functions --- vms/components/codec/codec.go | 127 +++++++++---------- vms/components/codec/codec_benchmark_test.go | 9 ++ vms/components/codec/codec_test.go | 1 - 3 files changed, 72 insertions(+), 65 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 182d8d2..560f2b7 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -93,15 +93,22 @@ func (c codec) Marshal(value interface{}) ([]byte, error) { if value == nil { return nil, errNil } - size, f, err := c.marshal(reflect.ValueOf(value)) + + funcs := make([]func(*wrappers.Packer) error, 512, 512) + size, _, err := c.marshal(reflect.ValueOf(value), 0, &funcs) if err != nil { return nil, err } p := &wrappers.Packer{MaxSize: size, Bytes: make([]byte, 0, size)} - if err := f(p); err != nil { - return nil, err + for _, f := range funcs { + if f == nil { + break + } else if err := f(p); err != nil { + return nil, err + } } + return p.Bytes, nil } @@ -111,14 +118,14 @@ func (c codec) Marshal(value interface{}) ([]byte, error) { // and returns the number of bytes it wrote. // When these functions are called in order, they write [value] to a byte slice. // 3) An error -func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) error, err error) { +func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.Packer) error) (size int, funcsWritten int, err error) { valueKind := value.Kind() // Case: Value can't be marshalled switch valueKind { case reflect.Interface, reflect.Ptr, reflect.Invalid: if value.IsNil() { // Can't marshal nil or nil pointers - return 0, nil, errNil + return 0, 0, errNil } } @@ -126,151 +133,150 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) switch valueKind { case reflect.Uint8: size = 1 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackByte(byte(value.Uint())) return p.Err } return case reflect.Int8: size = 1 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackByte(byte(value.Int())) return p.Err } return case reflect.Uint16: size = 2 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackShort(uint16(value.Uint())) return p.Err } return case reflect.Int16: size = 2 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackShort(uint16(value.Int())) return p.Err } return case reflect.Uint32: size = 4 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(uint32(value.Uint())) return p.Err } return case reflect.Int32: size = 4 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(uint32(value.Int())) return p.Err } return case reflect.Uint64: size = 8 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackLong(uint64(value.Uint())) return p.Err } return case reflect.Int64: size = 8 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackLong(uint64(value.Int())) return p.Err } return case reflect.String: + funcsWritten = 1 asStr := value.String() - size = len(asStr) + 2 - f = func(p *wrappers.Packer) error { + size = len(asStr) + wrappers.ShortLen + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackStr(asStr) return p.Err } return case reflect.Bool: size = 1 - f = func(p *wrappers.Packer) error { + funcsWritten = 1 + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackBool(value.Bool()) return p.Err } return case reflect.Uintptr, reflect.Ptr: - return c.marshal(value.Elem()) + return c.marshal(value.Elem(), index, funcs) case reflect.Interface: typeID, ok := c.typeToTypeID[reflect.TypeOf(value.Interface())] // Get the type ID of the value being marshaled if !ok { - return 0, nil, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(value.Interface()).String()) + return 0, 0, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(value.Interface()).String()) } - subsize, subfunc, subErr := c.marshal(reflect.ValueOf(value.Interface())) // TODO: Is this right? + (*funcs)[index] = nil + subsize, subFuncsWritten, subErr := c.marshal(reflect.ValueOf(value.Interface()), index+1, funcs) if subErr != nil { - return 0, nil, subErr + return 0, 0, subErr } size = 4 + subsize // 4 because we pack the type ID, a uint32 - f = func(p *wrappers.Packer) error { + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(typeID) if p.Err != nil { return p.Err } - return subfunc(p) + return nil } + funcsWritten = 1 + subFuncsWritten return case reflect.Slice: numElts := value.Len() // # elements in the slice/array. 0 if this slice is nil. if numElts > c.maxSliceLen { - return 0, nil, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) + return 0, 0, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } - subFuncs := make([]func(*wrappers.Packer) error, numElts, numElts) - size = wrappers.IntLen // for # elements + size = wrappers.IntLen // for # elements + subFuncsWritten := 0 for i := 0; i < numElts; i++ { // Process each element in the slice - subSize, subFunc, subErr := c.marshal(value.Index(i)) + subSize, n, subErr := c.marshal(value.Index(i), index+subFuncsWritten+1, funcs) if subErr != nil { - return 0, nil, subErr + return 0, 0, subErr } size += subSize - subFuncs[i] = subFunc + subFuncsWritten += n } - f = func(p *wrappers.Packer) error { + (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(uint32(numElts)) // pack # elements if p.Err != nil { return p.Err } - for _, f := range subFuncs { - if err := f(p); err != nil { - return err - } - } return nil } + funcsWritten = subFuncsWritten + 1 return case reflect.Array: numElts := value.Len() if numElts > c.maxSliceLen { - return 0, nil, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) + return 0, 0, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } size = 0 - subFuncs := make([]func(*wrappers.Packer) error, numElts, numElts) + funcsWritten = 0 for i := 0; i < numElts; i++ { // Process each element in the array - subSize, subFunc, subErr := c.marshal(value.Index(i)) + subSize, n, subErr := c.marshal(value.Index(i), index+funcsWritten, funcs) if subErr != nil { - return 0, nil, subErr + return 0, 0, subErr } size += subSize - subFuncs[i] = subFunc - } - - f = func(p *wrappers.Packer) error { - for _, f := range subFuncs { - if err := f(p); err != nil { - return err - } - } - return nil + funcsWritten += n } return case reflect.Struct: @@ -278,35 +284,28 @@ func (c codec) marshal(value reflect.Value) (size int, f func(*wrappers.Packer) numFields := t.NumField() size = 0 - subFuncs := make([]func(*wrappers.Packer) error, 0, numFields) + fieldsMarshalled := 0 + funcsWritten = 0 for i := 0; i < numFields; i++ { // Go through all fields of this struct field := t.Field(i) if !shouldSerialize(field) { // Skip fields we don't need to serialize continue } if unicode.IsLower(rune(field.Name[0])) { // Can only marshal exported fields - return 0, nil, fmt.Errorf("can't marshal unexported field %s", field.Name) + return 0, 0, fmt.Errorf("can't marshal unexported field %s", field.Name) } - fieldVal := value.Field(i) // The field we're serializing - subSize, subfunc, err := c.marshal(fieldVal) // Serialize the field + fieldVal := value.Field(i) // The field we're serializing + subSize, n, err := c.marshal(fieldVal, index+funcsWritten, funcs) // Serialize the field if err != nil { - return 0, nil, err + return 0, 0, err } + fieldsMarshalled++ size += subSize - subFuncs = append(subFuncs, subfunc) - } - - f = func(p *wrappers.Packer) error { - for _, f := range subFuncs { - if err := f(p); err != nil { - return err - } - } - return nil + funcsWritten += n } return default: - return 0, nil, errUnknownType + return 0, 0, errUnknownType } } diff --git a/vms/components/codec/codec_benchmark_test.go b/vms/components/codec/codec_benchmark_test.go index 8e6f9f7..ec4cc8c 100644 --- a/vms/components/codec/codec_benchmark_test.go +++ b/vms/components/codec/codec_benchmark_test.go @@ -53,3 +53,12 @@ func BenchmarkMarshalNonCodec(b *testing.B) { } } } + +func BenchmarkFoo(b *testing.B) { + arr := make([]int, 10000, 10000) + for n := 0; n < b.N; n++ { + for i := 0; i < 10000; i++ { + arr[i] = i + } + } +} diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index f42e827..74209eb 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -97,7 +97,6 @@ func TestStruct(t *testing.T) { if err != nil { t.Fatal(err) } - t.Logf("myStructBytes: %v", myStructBytes) myStructUnmarshaled := &myStruct{} err = codec.Unmarshal(myStructBytes, myStructUnmarshaled) From 954074abcc1e3a92d4b2b1c42f4e0265e4ecc9ca Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Fri, 12 Jun 2020 19:26:03 -0400 Subject: [PATCH 18/42] optimize by reducing amount of data stored on heap --- vms/components/codec/codec.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 560f2b7..be5acc1 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -134,64 +134,72 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P case reflect.Uint8: size = 1 funcsWritten = 1 + asByte := byte(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackByte(byte(value.Uint())) + p.PackByte(asByte) return p.Err } return case reflect.Int8: size = 1 funcsWritten = 1 + asByte := byte(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackByte(byte(value.Int())) + p.PackByte(asByte) return p.Err } return case reflect.Uint16: size = 2 funcsWritten = 1 + asShort := uint16(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackShort(uint16(value.Uint())) + p.PackShort(asShort) return p.Err } return case reflect.Int16: size = 2 funcsWritten = 1 + asShort := uint16(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackShort(uint16(value.Int())) + p.PackShort(asShort) return p.Err } return case reflect.Uint32: size = 4 funcsWritten = 1 + asInt := uint32(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(uint32(value.Uint())) + p.PackInt(asInt) return p.Err } return case reflect.Int32: size = 4 funcsWritten = 1 + asInt := uint32(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(uint32(value.Int())) + p.PackInt(asInt) return p.Err } return case reflect.Uint64: size = 8 funcsWritten = 1 + asInt := uint64(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackLong(uint64(value.Uint())) + p.PackLong(asInt) return p.Err } return case reflect.Int64: size = 8 funcsWritten = 1 + asInt := uint64(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackLong(uint64(value.Int())) + p.PackLong(asInt) return p.Err } return @@ -207,8 +215,9 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P case reflect.Bool: size = 1 funcsWritten = 1 + asBool := value.Bool() (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackBool(value.Bool()) + p.PackBool(asBool) return p.Err } return @@ -253,8 +262,9 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P subFuncsWritten += n } + numEltsAsUint32 := uint32(numElts) (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(uint32(numElts)) // pack # elements + p.PackInt(numEltsAsUint32) // pack # elements if p.Err != nil { return p.Err } From 760c32c4ac86bfe555657d431ad9b0c52d7d4eb5 Mon Sep 17 00:00:00 2001 From: Alex Willmer Date: Thu, 11 Jun 2020 20:50:13 +0100 Subject: [PATCH 19/42] main: Added database version & default network to -version --- main/params.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/main/params.go b/main/params.go index eef8e60..e285bc7 100644 --- a/main/params.go +++ b/main/params.go @@ -37,6 +37,7 @@ const ( var ( Config = node.Config{} Err error + defaultNetworkName = genesis.TestnetName defaultDbDir = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "db")) defaultStakingKeyPath = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "staking", "staker.key")) defaultStakingCertPath = os.ExpandEnv(filepath.Join("$HOME", ".gecko", "staking", "staker.crt")) @@ -169,7 +170,7 @@ func init() { version := fs.Bool("version", false, "If true, print version and quit") // NetworkID: - networkName := fs.String("network-id", genesis.TestnetName, "Network ID this node will connect to") + networkName := fs.String("network-id", defaultNetworkName, "Network ID this node will connect to") // Ava fees: fs.Uint64Var(&Config.AvaTxFee, "ava-tx-fee", 0, "Ava transaction fee, in $nAva") @@ -234,7 +235,15 @@ func init() { ferr := fs.Parse(os.Args[1:]) if *version { // If --version used, print version and exit - fmt.Println(node.Version.String()) + networkID, err := genesis.NetworkID(defaultNetworkName) + if errs.Add(err); err != nil { + return + } + networkGeneration := genesis.NetworkName(networkID) + fmt.Printf( + "%s [database=%s, network=%s/%s]\n", + node.Version, dbVersion, defaultNetworkName, networkGeneration, + ) os.Exit(0) } From d85ef87695ccc593dcca5c8081a88748c833d9d3 Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Sat, 13 Jun 2020 11:06:32 -0700 Subject: [PATCH 20/42] * Use Debug log level for all RPC calls * Use convention: [vm/api: function_name called...]. Ex: "Platform: SampleValidators called... --- api/ipcs/server.go | 2 ++ api/keystore/service.go | 10 ++++----- vms/avm/service.go | 34 ++++++++++++++--------------- vms/platformvm/service.go | 46 +++++++++++++++++++-------------------- 4 files changed, 47 insertions(+), 45 deletions(-) diff --git a/api/ipcs/server.go b/api/ipcs/server.go index 30bcc5d..f4be11b 100644 --- a/api/ipcs/server.go +++ b/api/ipcs/server.go @@ -61,6 +61,7 @@ type PublishBlockchainReply struct { // PublishBlockchain publishes the finalized accepted transactions from the blockchainID over the IPC func (ipc *IPCs) PublishBlockchain(r *http.Request, args *PublishBlockchainArgs, reply *PublishBlockchainReply) error { + ipc.log.Debug("IPCs: PublishBlockchain called with BlockchainID: %s", args.BlockchainID) chainID, err := ipc.chainManager.Lookup(args.BlockchainID) if err != nil { ipc.log.Error("unknown blockchainID: %s", err) @@ -116,6 +117,7 @@ type UnpublishBlockchainReply struct { // UnpublishBlockchain closes publishing of a blockchainID func (ipc *IPCs) UnpublishBlockchain(r *http.Request, args *UnpublishBlockchainArgs, reply *UnpublishBlockchainReply) error { + ipc.log.Debug("IPCs: UnpublishBlockchain called with BlockchainID: %s", args.BlockchainID) chainID, err := ipc.chainManager.Lookup(args.BlockchainID) if err != nil { ipc.log.Error("unknown blockchainID %s: %s", args.BlockchainID, err) diff --git a/api/keystore/service.go b/api/keystore/service.go index 16aca06..7006073 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -135,7 +135,7 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("CreateUser called with %.*s", maxUserPassLen, args.Username) + ks.log.Debug("Keystore: CreateUser called with %.*s", maxUserPassLen, args.Username) if len(args.Username) > maxUserPassLen || len(args.Password) > maxUserPassLen { return errUserPassMaxLength @@ -183,7 +183,7 @@ func (ks *Keystore) ListUsers(_ *http.Request, args *ListUsersArgs, reply *ListU ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("ListUsers called") + ks.log.Debug("Keystore: ListUsers called") reply.Users = []string{} @@ -211,7 +211,7 @@ func (ks *Keystore) ExportUser(_ *http.Request, args *ExportUserArgs, reply *Exp ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("ExportUser called for %s", args.Username) + ks.log.Debug("Keystore: ExportUser called for %s", args.Username) usr, err := ks.getUser(args.Username) if err != nil { @@ -264,7 +264,7 @@ func (ks *Keystore) ImportUser(r *http.Request, args *ImportUserArgs, reply *Imp ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("ImportUser called for %s", args.Username) + ks.log.Debug("Keystore: ImportUser called for %s", args.Username) if args.Username == "" { return errEmptyUsername @@ -324,7 +324,7 @@ func (ks *Keystore) DeleteUser(_ *http.Request, args *DeleteUserArgs, reply *Del ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("DeleteUser called with %s", args.Username) + ks.log.Debug("Keystore: DeleteUser called with %s", args.Username) if args.Username == "" { return errEmptyUsername diff --git a/vms/avm/service.go b/vms/avm/service.go index f71d607..4033be4 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -56,7 +56,7 @@ type IssueTxReply struct { // IssueTx attempts to issue a transaction into consensus func (service *Service) IssueTx(r *http.Request, args *IssueTxArgs, reply *IssueTxReply) error { - service.vm.ctx.Log.Verbo("IssueTx called with %s", args.Tx) + service.vm.ctx.Log.Debug("AVM: IssueTx called with %s", args.Tx) txID, err := service.vm.IssueTx(args.Tx.Bytes, nil) if err != nil { @@ -79,7 +79,7 @@ type GetTxStatusReply struct { // GetTxStatus returns the status of the specified transaction func (service *Service) GetTxStatus(r *http.Request, args *GetTxStatusArgs, reply *GetTxStatusReply) error { - service.vm.ctx.Log.Verbo("GetTxStatus called with %s", args.TxID) + service.vm.ctx.Log.Debug("AVM: GetTxStatus called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -106,7 +106,7 @@ type GetTxReply struct { // GetTx returns the specified transaction func (service *Service) GetTx(r *http.Request, args *GetTxArgs, reply *GetTxReply) error { - service.vm.ctx.Log.Verbo("GetTx called with %s", args.TxID) + service.vm.ctx.Log.Debug("AVM: GetTx called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -136,7 +136,7 @@ type GetUTXOsReply struct { // GetUTXOs creates an empty account with the name passed in func (service *Service) GetUTXOs(r *http.Request, args *GetUTXOsArgs, reply *GetUTXOsReply) error { - service.vm.ctx.Log.Verbo("GetUTXOs called with %s", args.Addresses) + service.vm.ctx.Log.Debug("AVM: GetUTXOs called with %s", args.Addresses) addrSet := ids.Set{} for _, addr := range args.Addresses { @@ -178,7 +178,7 @@ type GetAssetDescriptionReply struct { // GetAssetDescription creates an empty account with the name passed in func (service *Service) GetAssetDescription(_ *http.Request, args *GetAssetDescriptionArgs, reply *GetAssetDescriptionReply) error { - service.vm.ctx.Log.Verbo("GetAssetDescription called with %s", args.AssetID) + service.vm.ctx.Log.Debug("AVM: GetAssetDescription called with %s", args.AssetID) assetID, err := service.vm.Lookup(args.AssetID) if err != nil { @@ -222,7 +222,7 @@ type GetBalanceReply struct { // GetBalance returns the amount of an asset that an address at least partially owns func (service *Service) GetBalance(r *http.Request, args *GetBalanceArgs, reply *GetBalanceReply) error { - service.vm.ctx.Log.Verbo("GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) + service.vm.ctx.Log.Debug("AVM: GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) address, err := service.vm.Parse(args.Address) if err != nil { @@ -287,7 +287,7 @@ type GetAllBalancesReply struct { // Note that balances include assets that the address only _partially_ owns // (ie is one of several addresses specified in a multi-sig) func (service *Service) GetAllBalances(r *http.Request, args *GetAllBalancesArgs, reply *GetAllBalancesReply) error { - service.vm.ctx.Log.Verbo("GetAllBalances called with address: %s", args.Address) + service.vm.ctx.Log.Debug("AVM: GetAllBalances called with address: %s", args.Address) address, err := service.vm.Parse(args.Address) if err != nil { @@ -360,7 +360,7 @@ type CreateFixedCapAssetReply struct { // CreateFixedCapAsset returns ID of the newly created asset func (service *Service) CreateFixedCapAsset(r *http.Request, args *CreateFixedCapAssetArgs, reply *CreateFixedCapAssetReply) error { - service.vm.ctx.Log.Verbo("CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", + service.vm.ctx.Log.Debug("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", args.Name, args.Symbol, len(args.InitialHolders), @@ -445,7 +445,7 @@ type CreateVariableCapAssetReply struct { // CreateVariableCapAsset returns ID of the newly created asset func (service *Service) CreateVariableCapAsset(r *http.Request, args *CreateVariableCapAssetArgs, reply *CreateVariableCapAssetReply) error { - service.vm.ctx.Log.Verbo("CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", + service.vm.ctx.Log.Debug("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", args.Name, args.Symbol, len(args.MinterSets), @@ -523,7 +523,7 @@ type CreateAddressReply struct { // CreateAddress creates an address for the user [args.Username] func (service *Service) CreateAddress(r *http.Request, args *CreateAddressArgs, reply *CreateAddressReply) error { - service.vm.ctx.Log.Verbo("CreateAddress called for user '%s'", args.Username) + service.vm.ctx.Log.Debug("AVM: CreateAddress called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -603,7 +603,7 @@ type ExportKeyReply struct { // ExportKey returns a private key from the provided user func (service *Service) ExportKey(r *http.Request, args *ExportKeyArgs, reply *ExportKeyReply) error { - service.vm.ctx.Log.Verbo("ExportKey called for user '%s'", args.Username) + service.vm.ctx.Log.Debug("AVM: ExportKey called for user '%s'", args.Username) address, err := service.vm.Parse(args.Address) if err != nil { @@ -645,7 +645,7 @@ type ImportKeyReply struct { // ImportKey adds a private key to the provided user func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *ImportKeyReply) error { - service.vm.ctx.Log.Verbo("ImportKey called for user '%s'", args.Username) + service.vm.ctx.Log.Debug("AVM: ImportKey called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -692,7 +692,7 @@ type SendReply struct { // Send returns the ID of the newly created transaction func (service *Service) Send(r *http.Request, args *SendArgs, reply *SendReply) error { - service.vm.ctx.Log.Verbo("Send called with username: %s", args.Username) + service.vm.ctx.Log.Debug("AVM: Send called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount @@ -873,7 +873,7 @@ type CreateMintTxReply struct { // CreateMintTx returns the newly created unsigned transaction func (service *Service) CreateMintTx(r *http.Request, args *CreateMintTxArgs, reply *CreateMintTxReply) error { - service.vm.ctx.Log.Verbo("CreateMintTx called") + service.vm.ctx.Log.Debug("AVM: CreateMintTx called") if args.Amount == 0 { return errInvalidMintAmount @@ -990,7 +990,7 @@ type SignMintTxReply struct { // SignMintTx returns the newly signed transaction func (service *Service) SignMintTx(r *http.Request, args *SignMintTxArgs, reply *SignMintTxReply) error { - service.vm.ctx.Log.Verbo("SignMintTx called") + service.vm.ctx.Log.Debug("AVM: SignMintTx called") minter, err := service.vm.Parse(args.Minter) if err != nil { @@ -1116,7 +1116,7 @@ type ImportAVAReply struct { // The AVA must have already been exported from the P-Chain. // Returns the ID of the newly created atomic transaction func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, reply *ImportAVAReply) error { - service.vm.ctx.Log.Verbo("ImportAVA called with username: %s", args.Username) + service.vm.ctx.Log.Debug("AVM: ImportAVA called with username: %s", args.Username) toBytes, err := service.vm.Parse(args.To) if err != nil { @@ -1268,7 +1268,7 @@ type ExportAVAReply struct { // After this tx is accepted, the AVA must be imported to the P-chain with an importTx. // Returns the ID of the newly created atomic transaction func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, reply *ExportAVAReply) error { - service.vm.ctx.Log.Verbo("ExportAVA called with username: %s", args.Username) + service.vm.ctx.Log.Debug("AVM: ExportAVA called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index de2d41b..52010d1 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -234,7 +234,7 @@ type GetCurrentValidatorsReply struct { // GetCurrentValidators returns the list of current validators func (service *Service) GetCurrentValidators(_ *http.Request, args *GetCurrentValidatorsArgs, reply *GetCurrentValidatorsReply) error { - service.vm.Ctx.Log.Debug("GetCurrentValidators called") + service.vm.Ctx.Log.Debug("Platform: GetCurrentValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -298,7 +298,7 @@ type GetPendingValidatorsReply struct { // GetPendingValidators returns the list of current validators func (service *Service) GetPendingValidators(_ *http.Request, args *GetPendingValidatorsArgs, reply *GetPendingValidatorsReply) error { - service.vm.Ctx.Log.Debug("GetPendingValidators called") + service.vm.Ctx.Log.Debug("Platform: GetPendingValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -360,7 +360,7 @@ type SampleValidatorsReply struct { // SampleValidators returns a sampling of the list of current validators func (service *Service) SampleValidators(_ *http.Request, args *SampleValidatorsArgs, reply *SampleValidatorsReply) error { - service.vm.Ctx.Log.Debug("Sample called with {Size = %d}", args.Size) + service.vm.Ctx.Log.Debug("Platform: SampleValidators called with {Size = %d}", args.Size) if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -437,7 +437,7 @@ type ListAccountsReply struct { // ListAccounts lists all of the accounts controlled by [args.Username] func (service *Service) ListAccounts(_ *http.Request, args *ListAccountsArgs, reply *ListAccountsReply) error { - service.vm.Ctx.Log.Debug("listAccounts called for user '%s'", args.Username) + service.vm.Ctx.Log.Debug("Platform: ListAccounts called for user '%s'", args.Username) // db holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -499,7 +499,7 @@ type CreateAccountReply struct { // The account's ID is [privKey].PublicKey().Address(), where [privKey] is a // private key controlled by the user. func (service *Service) CreateAccount(_ *http.Request, args *CreateAccountArgs, reply *CreateAccountReply) error { - service.vm.Ctx.Log.Debug("createAccount called for user '%s'", args.Username) + service.vm.Ctx.Log.Debug("Platform: CreateAccount called for user '%s'", args.Username) // userDB holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -569,7 +569,7 @@ type AddDefaultSubnetValidatorArgs struct { // AddDefaultSubnetValidator returns an unsigned transaction to add a validator to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetValidator(_ *http.Request, args *AddDefaultSubnetValidatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("AddDefaultSubnetValidator called") + service.vm.Ctx.Log.Debug("Platform: AddDefaultSubnetValidator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -626,7 +626,7 @@ type AddDefaultSubnetDelegatorArgs struct { // to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetDelegator(_ *http.Request, args *AddDefaultSubnetDelegatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("AddDefaultSubnetDelegator called") + service.vm.Ctx.Log.Debug("Platform: AddDefaultSubnetDelegator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -741,7 +741,7 @@ type CreateSubnetArgs struct { // CreateSubnet returns an unsigned transaction to create a new subnet. // The unsigned transaction must be signed with the key of [args.Payer] func (service *Service) CreateSubnet(_ *http.Request, args *CreateSubnetArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("platform.createSubnet called") + service.vm.Ctx.Log.Debug("Platform: CreateSubnet called") switch { case args.PayerNonce == 0: @@ -796,7 +796,7 @@ type ExportAVAArgs struct { // The unsigned transaction must be signed with the key of the account exporting the AVA // and paying the transaction fee func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("platform.ExportAVA called") + service.vm.Ctx.Log.Debug("Platform: ExportAVA called") switch { case args.PayerNonce == 0: @@ -858,7 +858,7 @@ type SignResponse struct { // Sign [args.bytes] func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignResponse) error { - service.vm.Ctx.Log.Debug("sign called") + service.vm.Ctx.Log.Debug("Platform: Sign called") if args.Signer == "" { return errNilSigner @@ -915,7 +915,7 @@ func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignRespons // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Debug("signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("Platform: signAddDefaultSubnetValidatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetValidatorTx) @@ -938,7 +938,7 @@ func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetVali // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDelegatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetDelegatorTx, error) { - service.vm.Ctx.Log.Debug("signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("Platform: signAddDefaultSubnetDelegatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx) @@ -961,7 +961,7 @@ func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDele // Sign [xt] with [key] func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.PrivateKeySECP256K1R) (*CreateSubnetTx, error) { - service.vm.Ctx.Log.Debug("signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("Platform: signCreateSubnetTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedCreateSubnetTx) @@ -984,7 +984,7 @@ func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.Priva // Sign [tx] with [key] func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256K1R) (*ExportTx, error) { - service.vm.Ctx.Log.Debug("platform.signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("Platform: signExportTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedExportTx) @@ -1012,7 +1012,7 @@ func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256 // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signAddNonDefaultSubnetValidatorTx(tx *addNonDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addNonDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Debug("signAddNonDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("Platform: signAddNonDefaultSubnetValidatorTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedAddNonDefaultSubnetValidatorTx) @@ -1075,7 +1075,7 @@ type ImportAVAArgs struct { // The AVA must have already been exported from the X-Chain. // The unsigned transaction must be signed with the key of the tx fee payer. func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response *SignResponse) error { - service.vm.Ctx.Log.Debug("platform.ImportAVA called") + service.vm.Ctx.Log.Debug("Platform: ImportAVA called") switch { case args.To == "": @@ -1203,7 +1203,7 @@ func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signCreateChainTx(tx *CreateChainTx, key *crypto.PrivateKeySECP256K1R) (*CreateChainTx, error) { - service.vm.Ctx.Log.Debug("signCreateChainTx called") + service.vm.Ctx.Log.Debug("Platform: signCreateChainTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedCreateChainTx) @@ -1263,7 +1263,7 @@ type IssueTxResponse struct { // IssueTx issues the transaction [args.Tx] to the network func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *IssueTxResponse) error { - service.vm.Ctx.Log.Debug("issueTx called") + service.vm.Ctx.Log.Debug("Platform: IssueTx called") genTx := genericTx{} if err := Codec.Unmarshal(args.Tx.Bytes, &genTx); err != nil { @@ -1327,7 +1327,7 @@ type CreateBlockchainArgs struct { // CreateBlockchain returns an unsigned transaction to create a new blockchain // Must be signed with the Subnet's control keys and with a key that pays the transaction fee before issuance func (service *Service) CreateBlockchain(_ *http.Request, args *CreateBlockchainArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("createBlockchain called") + service.vm.Ctx.Log.Debug("Platform: CreateBlockchain called") switch { case args.PayerNonce == 0: @@ -1410,7 +1410,7 @@ type GetBlockchainStatusReply struct { // GetBlockchainStatus gets the status of a blockchain with the ID [args.BlockchainID]. func (service *Service) GetBlockchainStatus(_ *http.Request, args *GetBlockchainStatusArgs, reply *GetBlockchainStatusReply) error { - service.vm.Ctx.Log.Debug("getBlockchainStatus called") + service.vm.Ctx.Log.Debug("Platform: GetBlockchainStatus called") switch { case args.BlockchainID == "": @@ -1490,7 +1490,7 @@ type ValidatedByResponse struct { // ValidatedBy returns the ID of the Subnet that validates [args.BlockchainID] func (service *Service) ValidatedBy(_ *http.Request, args *ValidatedByArgs, response *ValidatedByResponse) error { - service.vm.Ctx.Log.Debug("validatedBy called") + service.vm.Ctx.Log.Debug("Platform: ValidatedBy called") switch { case args.BlockchainID == "": @@ -1522,7 +1522,7 @@ type ValidatesResponse struct { // Validates returns the IDs of the blockchains validated by [args.SubnetID] func (service *Service) Validates(_ *http.Request, args *ValidatesArgs, response *ValidatesResponse) error { - service.vm.Ctx.Log.Debug("validates called") + service.vm.Ctx.Log.Debug("Platform: Validates called") switch { case args.SubnetID == "": @@ -1576,7 +1576,7 @@ type GetBlockchainsResponse struct { // GetBlockchains returns all of the blockchains that exist func (service *Service) GetBlockchains(_ *http.Request, args *struct{}, response *GetBlockchainsResponse) error { - service.vm.Ctx.Log.Debug("getBlockchains called") + service.vm.Ctx.Log.Debug("Platform: GetBlockchains called") chains, err := service.vm.getChains(service.vm.DB) if err != nil { From f6cabee51b2cf9e897798de8bd8edc585ff49e30 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 10:56:43 -0400 Subject: [PATCH 21/42] cache serializable fields of struct types; change codec methods to be on pointer type; change variable names; change benchmark toinclude both marshaling and unmarshaling --- vms/components/codec/codec.go | 136 ++++++++++--------- vms/components/codec/codec_benchmark_test.go | 11 +- 2 files changed, 82 insertions(+), 65 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index be5acc1..29cfaef 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -41,6 +41,8 @@ type codec struct { typeIDToType map[uint32]reflect.Type typeToTypeID map[reflect.Type]uint32 + + serializedFields map[reflect.Type][]int } // Codec marshals and unmarshals @@ -52,7 +54,7 @@ type Codec interface { // New returns a new codec func New(maxSize, maxSliceLen int) Codec { - return codec{ + return &codec{ maxSize: maxSize, maxSliceLen: maxSliceLen, typeIDToType: map[uint32]reflect.Type{}, @@ -65,7 +67,7 @@ func NewDefault() Codec { return New(defaultMaxSize, defaultMaxSliceLength) } // RegisterType is used to register types that may be unmarshaled into an interface // [val] is a value of the type being registered -func (c codec) RegisterType(val interface{}) error { +func (c *codec) RegisterType(val interface{}) error { valType := reflect.TypeOf(val) if _, exists := c.typeToTypeID[valType]; exists { return fmt.Errorf("type %v has already been registered", valType) @@ -89,7 +91,7 @@ func (c codec) RegisterType(val interface{}) error { // 8) nil slices are marshaled as empty slices // To marshal an interface, [value] must be a pointer to the interface -func (c codec) Marshal(value interface{}) ([]byte, error) { +func (c *codec) Marshal(value interface{}) ([]byte, error) { if value == nil { return nil, errNil } @@ -118,7 +120,7 @@ func (c codec) Marshal(value interface{}) ([]byte, error) { // and returns the number of bytes it wrote. // When these functions are called in order, they write [value] to a byte slice. // 3) An error -func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.Packer) error) (size int, funcsWritten int, err error) { +func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.Packer) error) (size int, funcsWritten int, err error) { valueKind := value.Kind() // Case: Value can't be marshalled @@ -238,10 +240,7 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P size = 4 + subsize // 4 because we pack the type ID, a uint32 (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(typeID) - if p.Err != nil { - return p.Err - } - return nil + return p.Err } funcsWritten = 1 + subFuncsWritten return @@ -265,10 +264,7 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P numEltsAsUint32 := uint32(numElts) (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(numEltsAsUint32) // pack # elements - if p.Err != nil { - return p.Err - } - return nil + return p.Err } funcsWritten = subFuncsWritten + 1 return @@ -291,20 +287,17 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P return case reflect.Struct: t := value.Type() - numFields := t.NumField() size = 0 fieldsMarshalled := 0 funcsWritten = 0 - for i := 0; i < numFields; i++ { // Go through all fields of this struct - field := t.Field(i) - if !shouldSerialize(field) { // Skip fields we don't need to serialize - continue - } - if unicode.IsLower(rune(field.Name[0])) { // Can only marshal exported fields - return 0, 0, fmt.Errorf("can't marshal unexported field %s", field.Name) - } - fieldVal := value.Field(i) // The field we're serializing + serializedFields, subErr := c.getSerializedFieldIndices(t) + if subErr != nil { + return 0, 0, subErr + } + + for _, f := range serializedFields { // Go through all fields of this struct + fieldVal := value.Field(f) // The field we're serializing subSize, n, err := c.marshal(fieldVal, index+funcsWritten, funcs) // Serialize the field if err != nil { return 0, 0, err @@ -321,7 +314,7 @@ func (c codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.P // Unmarshal unmarshals [bytes] into [dest], where // [dest] must be a pointer or interface -func (c codec) Unmarshal(bytes []byte, dest interface{}) error { +func (c *codec) Unmarshal(bytes []byte, dest interface{}) error { switch { case len(bytes) > c.maxSize: return errSliceTooLarge @@ -343,92 +336,90 @@ func (c codec) Unmarshal(bytes []byte, dest interface{}) error { return nil } -// Unmarshal bytes from [bytes] into [field] -// [field] must be addressable -func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { - kind := field.Kind() - switch kind { +// Unmarshal from [bytes] into [value]. [value] must be addressable +func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { + switch value.Kind() { case reflect.Uint8: b := p.UnpackByte() if p.Err != nil { return p.Err } - field.SetUint(uint64(b)) + value.SetUint(uint64(b)) return nil case reflect.Int8: b := p.UnpackByte() if p.Err != nil { return p.Err } - field.SetInt(int64(b)) + value.SetInt(int64(b)) return nil case reflect.Uint16: b := p.UnpackShort() if p.Err != nil { return p.Err } - field.SetUint(uint64(b)) + value.SetUint(uint64(b)) return nil case reflect.Int16: b := p.UnpackShort() if p.Err != nil { return p.Err } - field.SetInt(int64(b)) + value.SetInt(int64(b)) return nil case reflect.Uint32: b := p.UnpackInt() if p.Err != nil { return p.Err } - field.SetUint(uint64(b)) + value.SetUint(uint64(b)) return nil case reflect.Int32: b := p.UnpackInt() if p.Err != nil { return p.Err } - field.SetInt(int64(b)) + value.SetInt(int64(b)) return nil case reflect.Uint64: b := p.UnpackLong() if p.Err != nil { return p.Err } - field.SetUint(uint64(b)) + value.SetUint(uint64(b)) return nil case reflect.Int64: b := p.UnpackLong() if p.Err != nil { return p.Err } - field.SetInt(int64(b)) + value.SetInt(int64(b)) return nil case reflect.Bool: b := p.UnpackBool() if p.Err != nil { return p.Err } - field.SetBool(b) + value.SetBool(b) return nil case reflect.Slice: numElts := int(p.UnpackInt()) if p.Err != nil { return p.Err } - // set [field] to be a slice of the appropriate type/capacity (right now [field] is nil) - slice := reflect.MakeSlice(field.Type(), numElts, numElts) - field.Set(slice) + // set [value] to be a slice of the appropriate type/capacity (right now [value] is nil) + slice := reflect.MakeSlice(value.Type(), numElts, numElts) + value.Set(slice) // Unmarshal each element into the appropriate index of the slice for i := 0; i < numElts; i++ { - if err := c.unmarshal(p, field.Index(i)); err != nil { + if err := c.unmarshal(p, value.Index(i)); err != nil { return err } } return nil case reflect.Array: - for i := 0; i < field.Len(); i++ { - if err := c.unmarshal(p, field.Index(i)); err != nil { + for i := 0; i < value.Len(); i++ { + if err := c.unmarshal(p, value.Index(i)); err != nil { return err } } @@ -438,7 +429,7 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { if p.Err != nil { return p.Err } - field.SetString(str) + value.SetString(str) return nil case reflect.Interface: typeID := p.UnpackInt() // Get the type ID @@ -451,31 +442,28 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { return errUnmarshalUnregisteredType } // Ensure struct actually does implement the interface - fieldType := field.Type() - if !typ.Implements(fieldType) { - return fmt.Errorf("%s does not implement interface %s", typ, fieldType) + valueType := value.Type() + if !typ.Implements(valueType) { + return fmt.Errorf("%s does not implement interface %s", typ, valueType) } concreteInstancePtr := reflect.New(typ) // instance of the proper type // Unmarshal into the struct if err := c.unmarshal(p, concreteInstancePtr.Elem()); err != nil { return err } - // And assign the filled struct to the field - field.Set(concreteInstancePtr.Elem()) + // And assign the filled struct to the value + value.Set(concreteInstancePtr.Elem()) return nil case reflect.Struct: // Type of this struct - structType := reflect.TypeOf(field.Interface()) + t := reflect.TypeOf(value.Interface()) + serializedFieldIndices, err := c.getSerializedFieldIndices(t) + if err != nil { + return err + } // Go through all the fields and umarshal into each - for i := 0; i < structType.NumField(); i++ { - structField := structType.Field(i) - if !shouldSerialize(structField) { // Skip fields we don't need to unmarshal - continue - } - if unicode.IsLower(rune(structField.Name[0])) { // Only unmarshal into exported field - return errUnmarshalUnexportedField - } - field := field.Field(i) // Get the field + for _, index := range serializedFieldIndices { + field := value.Field(index) // Get the field if err := c.unmarshal(p, field); err != nil { // Unmarshal into the field return err } @@ -483,7 +471,7 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { return nil case reflect.Ptr: // Get the type this pointer points to - underlyingType := field.Type().Elem() + underlyingType := value.Type().Elem() // Create a new pointer to a new value of the underlying type underlyingValue := reflect.New(underlyingType) // Fill the value @@ -491,7 +479,7 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { return err } // Assign to the top-level struct's member - field.Set(underlyingValue) + value.Set(underlyingValue) return nil case reflect.Invalid: return errNil @@ -500,7 +488,27 @@ func (c codec) unmarshal(p *wrappers.Packer, field reflect.Value) error { } } -// Returns true iff [field] should be serialized -func shouldSerialize(field reflect.StructField) bool { - return field.Tag.Get("serialize") == "true" +// Returns the indices of the serializable fields of [t], which is a struct type +// Returns an error if a field has tag "serialize: true" but the field is unexported +func (c *codec) getSerializedFieldIndices(t reflect.Type) ([]int, error) { + if c.serializedFields == nil { + c.serializedFields = make(map[reflect.Type][]int) + } + if serializedFields, ok := c.serializedFields[t]; ok { + return serializedFields, nil + } + numFields := t.NumField() + serializedFields := make([]int, 0, numFields) + for i := 0; i < numFields; i++ { // Go through all fields of this struct + field := t.Field(i) + if field.Tag.Get("serialize") != "true" { // Skip fields we don't need to serialize + continue + } + if unicode.IsLower(rune(field.Name[0])) { // Can only marshal exported fields + return []int{}, fmt.Errorf("can't marshal unexported field %s", field.Name) + } + serializedFields = append(serializedFields, i) + } + c.serializedFields[t] = serializedFields + return serializedFields, nil } diff --git a/vms/components/codec/codec_benchmark_test.go b/vms/components/codec/codec_benchmark_test.go index ec4cc8c..25af563 100644 --- a/vms/components/codec/codec_benchmark_test.go +++ b/vms/components/codec/codec_benchmark_test.go @@ -35,13 +35,22 @@ func BenchmarkMarshal(b *testing.B) { }, MyPointer: &temp, } + var unmarshaledMyStructInstance myStruct codec := NewDefault() codec.RegisterType(&MyInnerStruct{}) // Register the types that may be unmarshaled into interfaces codec.RegisterType(&MyInnerStruct2{}) + codec.Marshal(myStructInstance) // warm up serializedFields cache b.ResetTimer() for n := 0; n < b.N; n++ { - codec.Marshal(myStructInstance) + bytes, err := codec.Marshal(myStructInstance) + if err != nil { + b.Fatal(err) + } + if err := codec.Unmarshal(bytes, &unmarshaledMyStructInstance); err != nil { + b.Fatal(err) + } + } } From 7b5b3d1f1cf879c1e6a991b95a7deafde0b958d7 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 11:53:19 -0400 Subject: [PATCH 22/42] more optimizations/cleanup --- vms/components/codec/codec.go | 42 ++++++++++++++++------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 29cfaef..5000a6a 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -42,7 +42,7 @@ type codec struct { typeIDToType map[uint32]reflect.Type typeToTypeID map[reflect.Type]uint32 - serializedFields map[reflect.Type][]int + serializedFieldIndices map[reflect.Type][]int } // Codec marshals and unmarshals @@ -55,10 +55,11 @@ type Codec interface { // New returns a new codec func New(maxSize, maxSliceLen int) Codec { return &codec{ - maxSize: maxSize, - maxSliceLen: maxSliceLen, - typeIDToType: map[uint32]reflect.Type{}, - typeToTypeID: map[reflect.Type]uint32{}, + maxSize: maxSize, + maxSliceLen: maxSliceLen, + typeIDToType: map[uint32]reflect.Type{}, + typeToTypeID: map[reflect.Type]uint32{}, + serializedFieldIndices: map[reflect.Type][]int{}, } } @@ -226,13 +227,14 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. case reflect.Uintptr, reflect.Ptr: return c.marshal(value.Elem(), index, funcs) case reflect.Interface: - typeID, ok := c.typeToTypeID[reflect.TypeOf(value.Interface())] // Get the type ID of the value being marshaled + underlyingValue := value.Interface() + typeID, ok := c.typeToTypeID[reflect.TypeOf(underlyingValue)] // Get the type ID of the value being marshaled if !ok { - return 0, 0, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(value.Interface()).String()) + return 0, 0, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(underlyingValue).String()) } (*funcs)[index] = nil - subsize, subFuncsWritten, subErr := c.marshal(reflect.ValueOf(value.Interface()), index+1, funcs) + subsize, subFuncsWritten, subErr := c.marshal(value.Elem(), index+1, funcs) if subErr != nil { return 0, 0, subErr } @@ -408,8 +410,7 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { return p.Err } // set [value] to be a slice of the appropriate type/capacity (right now [value] is nil) - slice := reflect.MakeSlice(value.Type(), numElts, numElts) - value.Set(slice) + value.Set(reflect.MakeSlice(value.Type(), numElts, numElts)) // Unmarshal each element into the appropriate index of the slice for i := 0; i < numElts; i++ { if err := c.unmarshal(p, value.Index(i)); err != nil { @@ -425,12 +426,8 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { } return nil case reflect.String: - str := p.UnpackStr() - if p.Err != nil { - return p.Err - } - value.SetString(str) - return nil + value.SetString(p.UnpackStr()) + return p.Err case reflect.Interface: typeID := p.UnpackInt() // Get the type ID if p.Err != nil { @@ -456,15 +453,14 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { return nil case reflect.Struct: // Type of this struct - t := reflect.TypeOf(value.Interface()) + t := value.Type() serializedFieldIndices, err := c.getSerializedFieldIndices(t) if err != nil { return err } // Go through all the fields and umarshal into each for _, index := range serializedFieldIndices { - field := value.Field(index) // Get the field - if err := c.unmarshal(p, field); err != nil { // Unmarshal into the field + if err := c.unmarshal(p, value.Field(index)); err != nil { // Unmarshal into the field return err } } @@ -491,10 +487,10 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { // Returns the indices of the serializable fields of [t], which is a struct type // Returns an error if a field has tag "serialize: true" but the field is unexported func (c *codec) getSerializedFieldIndices(t reflect.Type) ([]int, error) { - if c.serializedFields == nil { - c.serializedFields = make(map[reflect.Type][]int) + if c.serializedFieldIndices == nil { + c.serializedFieldIndices = make(map[reflect.Type][]int) } - if serializedFields, ok := c.serializedFields[t]; ok { + if serializedFields, ok := c.serializedFieldIndices[t]; ok { return serializedFields, nil } numFields := t.NumField() @@ -509,6 +505,6 @@ func (c *codec) getSerializedFieldIndices(t reflect.Type) ([]int, error) { } serializedFields = append(serializedFields, i) } - c.serializedFields[t] = serializedFields + c.serializedFieldIndices[t] = serializedFields return serializedFields, nil } From ee1cf620a18c1c12c6284969cb77c99d4fb4fe58 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 12:06:39 -0400 Subject: [PATCH 23/42] cleanup --- vms/components/codec/codec.go | 88 ++++++++++------------------------- 1 file changed, 25 insertions(+), 63 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 5000a6a..cda5fba 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -342,68 +342,32 @@ func (c *codec) Unmarshal(bytes []byte, dest interface{}) error { func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { switch value.Kind() { case reflect.Uint8: - b := p.UnpackByte() - if p.Err != nil { - return p.Err - } - value.SetUint(uint64(b)) - return nil + value.SetUint(uint64(p.UnpackByte())) + return p.Err case reflect.Int8: - b := p.UnpackByte() - if p.Err != nil { - return p.Err - } - value.SetInt(int64(b)) - return nil + value.SetInt(int64(p.UnpackByte())) + return p.Err case reflect.Uint16: - b := p.UnpackShort() - if p.Err != nil { - return p.Err - } - value.SetUint(uint64(b)) - return nil + value.SetUint(uint64(p.UnpackShort())) + return p.Err case reflect.Int16: - b := p.UnpackShort() - if p.Err != nil { - return p.Err - } - value.SetInt(int64(b)) - return nil + value.SetInt(int64(p.UnpackShort())) + return p.Err case reflect.Uint32: - b := p.UnpackInt() - if p.Err != nil { - return p.Err - } - value.SetUint(uint64(b)) - return nil + value.SetUint(uint64(p.UnpackInt())) + return p.Err case reflect.Int32: - b := p.UnpackInt() - if p.Err != nil { - return p.Err - } - value.SetInt(int64(b)) - return nil + value.SetInt(int64(p.UnpackInt())) + return p.Err case reflect.Uint64: - b := p.UnpackLong() - if p.Err != nil { - return p.Err - } - value.SetUint(uint64(b)) - return nil + value.SetUint(uint64(p.UnpackLong())) + return p.Err case reflect.Int64: - b := p.UnpackLong() - if p.Err != nil { - return p.Err - } - value.SetInt(int64(b)) - return nil + value.SetInt(int64(p.UnpackLong())) + return p.Err case reflect.Bool: - b := p.UnpackBool() - if p.Err != nil { - return p.Err - } - value.SetBool(b) - return nil + value.SetBool(p.UnpackBool()) + return p.Err case reflect.Slice: numElts := int(p.UnpackInt()) if p.Err != nil { @@ -433,28 +397,26 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { if p.Err != nil { return p.Err } - // Get a struct that implements the interface + // Get a type that implements the interface typ, ok := c.typeIDToType[typeID] if !ok { return errUnmarshalUnregisteredType } - // Ensure struct actually does implement the interface - valueType := value.Type() - if !typ.Implements(valueType) { + // Ensure type actually does implement the interface + if valueType := value.Type(); !typ.Implements(valueType) { return fmt.Errorf("%s does not implement interface %s", typ, valueType) } - concreteInstancePtr := reflect.New(typ) // instance of the proper type + concreteInstance := reflect.New(typ).Elem() // instance of the proper type // Unmarshal into the struct - if err := c.unmarshal(p, concreteInstancePtr.Elem()); err != nil { + if err := c.unmarshal(p, concreteInstance); err != nil { return err } // And assign the filled struct to the value - value.Set(concreteInstancePtr.Elem()) + value.Set(concreteInstance) return nil case reflect.Struct: // Type of this struct - t := value.Type() - serializedFieldIndices, err := c.getSerializedFieldIndices(t) + serializedFieldIndices, err := c.getSerializedFieldIndices(value.Type()) if err != nil { return err } From 9c4cfecf4e5bdd1b0119bd28bf2c360f9cd1c07b Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 12:23:05 -0400 Subject: [PATCH 24/42] pack pointer to string instead of string...halves memory footprint --- utils/wrappers/packing.go | 10 ++++++++++ vms/components/codec/codec.go | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/utils/wrappers/packing.go b/utils/wrappers/packing.go index 1038852..3759c63 100644 --- a/utils/wrappers/packing.go +++ b/utils/wrappers/packing.go @@ -284,6 +284,16 @@ func (p *Packer) PackStr(str string) { p.PackFixedBytes([]byte(str)) } +// PackStrPtr appends a string to the byte array +func (p *Packer) PackStrPtr(str *string) { + strSize := len(*str) + if strSize > MaxStringLen { + p.Add(errInvalidInput) + } + p.PackShort(uint16(strSize)) + p.PackFixedBytes([]byte(*str)) +} + // UnpackStr unpacks a string from the byte array func (p *Packer) UnpackStr() string { strSize := p.UnpackShort() diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index cda5fba..094f9e1 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -127,12 +127,11 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. // Case: Value can't be marshalled switch valueKind { case reflect.Interface, reflect.Ptr, reflect.Invalid: - if value.IsNil() { // Can't marshal nil or nil pointers + if value.IsNil() { // Can't marshal nil (except nil slices) return 0, 0, errNil } } - // Case: Value is of known size; return its byte repr. switch valueKind { case reflect.Uint8: size = 1 @@ -211,7 +210,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. asStr := value.String() size = len(asStr) + wrappers.ShortLen (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackStr(asStr) + p.PackStrPtr(&asStr) return p.Err } return From d1796c8b0bd0bb7f9122b50c371442c561df7eb3 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 18:15:44 -0400 Subject: [PATCH 25/42] cleanup; revert string packing method --- utils/wrappers/packing.go | 10 -------- vms/components/codec/codec.go | 46 +++++++++++++++-------------------- 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/utils/wrappers/packing.go b/utils/wrappers/packing.go index 3759c63..1038852 100644 --- a/utils/wrappers/packing.go +++ b/utils/wrappers/packing.go @@ -284,16 +284,6 @@ func (p *Packer) PackStr(str string) { p.PackFixedBytes([]byte(str)) } -// PackStrPtr appends a string to the byte array -func (p *Packer) PackStrPtr(str *string) { - strSize := len(*str) - if strSize > MaxStringLen { - p.Add(errInvalidInput) - } - p.PackShort(uint16(strSize)) - p.PackFixedBytes([]byte(*str)) -} - // UnpackStr unpacks a string from the byte array func (p *Packer) UnpackStr() string { strSize := p.UnpackShort() diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 094f9e1..ce488e7 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -97,7 +97,7 @@ func (c *codec) Marshal(value interface{}) ([]byte, error) { return nil, errNil } - funcs := make([]func(*wrappers.Packer) error, 512, 512) + funcs := make([]func(*wrappers.Packer) error, 256, 256) size, _, err := c.marshal(reflect.ValueOf(value), 0, &funcs) if err != nil { return nil, err @@ -134,7 +134,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. switch valueKind { case reflect.Uint8: - size = 1 + size = wrappers.ByteLen funcsWritten = 1 asByte := byte(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -143,7 +143,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Int8: - size = 1 + size = wrappers.ByteLen funcsWritten = 1 asByte := byte(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -152,7 +152,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Uint16: - size = 2 + size = wrappers.ShortLen funcsWritten = 1 asShort := uint16(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -161,7 +161,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Int16: - size = 2 + size = wrappers.ShortLen funcsWritten = 1 asShort := uint16(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -170,7 +170,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Uint32: - size = 4 + size = wrappers.IntLen funcsWritten = 1 asInt := uint32(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -179,7 +179,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Int32: - size = 4 + size = wrappers.IntLen funcsWritten = 1 asInt := uint32(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -188,7 +188,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Uint64: - size = 8 + size = wrappers.LongLen funcsWritten = 1 asInt := uint64(value.Uint()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -197,7 +197,7 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Int64: - size = 8 + size = wrappers.LongLen funcsWritten = 1 asInt := uint64(value.Int()) (*funcs)[index] = func(p *wrappers.Packer) error { @@ -206,16 +206,17 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.String: + // Note: it actually saves memory allocations to not do s := value.String() + // and use s in place of value.String(). Hence we don't do that. funcsWritten = 1 - asStr := value.String() - size = len(asStr) + wrappers.ShortLen + size = len(value.String()) + wrappers.ShortLen (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackStrPtr(&asStr) + p.PackStr(value.String()) return p.Err } return case reflect.Bool: - size = 1 + size = wrappers.BoolLen funcsWritten = 1 asBool := value.Bool() (*funcs)[index] = func(p *wrappers.Packer) error { @@ -232,13 +233,11 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. return 0, 0, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(underlyingValue).String()) } - (*funcs)[index] = nil subsize, subFuncsWritten, subErr := c.marshal(value.Elem(), index+1, funcs) if subErr != nil { return 0, 0, subErr } - - size = 4 + subsize // 4 because we pack the type ID, a uint32 + size = wrappers.IntLen + subsize (*funcs)[index] = func(p *wrappers.Packer) error { p.PackInt(typeID) return p.Err @@ -287,23 +286,18 @@ func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers. } return case reflect.Struct: - t := value.Type() - - size = 0 - fieldsMarshalled := 0 - funcsWritten = 0 - serializedFields, subErr := c.getSerializedFieldIndices(t) + serializedFields, subErr := c.getSerializedFieldIndices(value.Type()) if subErr != nil { return 0, 0, subErr } - for _, f := range serializedFields { // Go through all fields of this struct - fieldVal := value.Field(f) // The field we're serializing - subSize, n, err := c.marshal(fieldVal, index+funcsWritten, funcs) // Serialize the field + size = 0 + funcsWritten = 0 + for _, fieldIndex := range serializedFields { // Go through all fields of this struct + subSize, n, err := c.marshal(value.Field(fieldIndex), index+funcsWritten, funcs) // Serialize the field if err != nil { return 0, 0, err } - fieldsMarshalled++ size += subSize funcsWritten += n } From b269f8cfb0b7a3a66ca31570cf27108eb1cbf1a2 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 14 Jun 2020 21:38:07 -0400 Subject: [PATCH 26/42] marshal writes directly to packer rather than creating array of functions --- vms/components/codec/codec.go | 196 +++++++++------------------------- 1 file changed, 52 insertions(+), 144 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index ce488e7..8e223f9 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -97,21 +97,11 @@ func (c *codec) Marshal(value interface{}) ([]byte, error) { return nil, errNil } - funcs := make([]func(*wrappers.Packer) error, 256, 256) - size, _, err := c.marshal(reflect.ValueOf(value), 0, &funcs) - if err != nil { + p := &wrappers.Packer{MaxSize: 512, Bytes: make([]byte, 0, 512)} + if err := c.marshal(reflect.ValueOf(value), p); err != nil { return nil, err } - p := &wrappers.Packer{MaxSize: size, Bytes: make([]byte, 0, size)} - for _, f := range funcs { - if f == nil { - break - } else if err := f(p); err != nil { - return nil, err - } - } - return p.Bytes, nil } @@ -121,189 +111,107 @@ func (c *codec) Marshal(value interface{}) ([]byte, error) { // and returns the number of bytes it wrote. // When these functions are called in order, they write [value] to a byte slice. // 3) An error -func (c *codec) marshal(value reflect.Value, index int, funcs *[]func(*wrappers.Packer) error) (size int, funcsWritten int, err error) { +func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { valueKind := value.Kind() // Case: Value can't be marshalled switch valueKind { case reflect.Interface, reflect.Ptr, reflect.Invalid: if value.IsNil() { // Can't marshal nil (except nil slices) - return 0, 0, errNil + return errNil } } switch valueKind { case reflect.Uint8: - size = wrappers.ByteLen - funcsWritten = 1 - asByte := byte(value.Uint()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackByte(asByte) - return p.Err - } - return + p.PackByte(uint8(value.Uint())) + return p.Err case reflect.Int8: - size = wrappers.ByteLen - funcsWritten = 1 - asByte := byte(value.Int()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackByte(asByte) - return p.Err - } - return + p.PackByte(uint8(value.Int())) + return p.Err case reflect.Uint16: - size = wrappers.ShortLen - funcsWritten = 1 - asShort := uint16(value.Uint()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackShort(asShort) - return p.Err - } - return + p.PackShort(uint16(value.Uint())) + return p.Err case reflect.Int16: - size = wrappers.ShortLen - funcsWritten = 1 - asShort := uint16(value.Int()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackShort(asShort) - return p.Err - } - return + p.PackShort(uint16(value.Int())) + return p.Err case reflect.Uint32: - size = wrappers.IntLen - funcsWritten = 1 - asInt := uint32(value.Uint()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(asInt) - return p.Err - } - return + p.PackInt(uint32(value.Uint())) + return p.Err case reflect.Int32: - size = wrappers.IntLen - funcsWritten = 1 - asInt := uint32(value.Int()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(asInt) - return p.Err - } - return + p.PackInt(uint32(value.Int())) + return p.Err case reflect.Uint64: - size = wrappers.LongLen - funcsWritten = 1 - asInt := uint64(value.Uint()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackLong(asInt) - return p.Err - } - return + p.PackLong(value.Uint()) + return p.Err case reflect.Int64: - size = wrappers.LongLen - funcsWritten = 1 - asInt := uint64(value.Int()) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackLong(asInt) - return p.Err - } - return + p.PackLong(uint64(value.Int())) + return p.Err case reflect.String: - // Note: it actually saves memory allocations to not do s := value.String() - // and use s in place of value.String(). Hence we don't do that. - funcsWritten = 1 - size = len(value.String()) + wrappers.ShortLen - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackStr(value.String()) - return p.Err - } - return + p.PackStr(value.String()) + return p.Err case reflect.Bool: - size = wrappers.BoolLen - funcsWritten = 1 - asBool := value.Bool() - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackBool(asBool) - return p.Err - } - return + p.PackBool(value.Bool()) + return p.Err case reflect.Uintptr, reflect.Ptr: - return c.marshal(value.Elem(), index, funcs) + return c.marshal(value.Elem(), p) case reflect.Interface: underlyingValue := value.Interface() typeID, ok := c.typeToTypeID[reflect.TypeOf(underlyingValue)] // Get the type ID of the value being marshaled if !ok { - return 0, 0, fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(underlyingValue).String()) + return fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(underlyingValue).String()) } - - subsize, subFuncsWritten, subErr := c.marshal(value.Elem(), index+1, funcs) - if subErr != nil { - return 0, 0, subErr - } - size = wrappers.IntLen + subsize - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(typeID) + p.PackInt(typeID) + if p.Err != nil { return p.Err } - funcsWritten = 1 + subFuncsWritten - return + if err := c.marshal(value.Elem(), p); err != nil { + return err + } + return p.Err case reflect.Slice: numElts := value.Len() // # elements in the slice/array. 0 if this slice is nil. if numElts > c.maxSliceLen { - return 0, 0, fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) + return fmt.Errorf("slice length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } - - size = wrappers.IntLen // for # elements - subFuncsWritten := 0 - for i := 0; i < numElts; i++ { // Process each element in the slice - subSize, n, subErr := c.marshal(value.Index(i), index+subFuncsWritten+1, funcs) - if subErr != nil { - return 0, 0, subErr - } - size += subSize - subFuncsWritten += n - } - - numEltsAsUint32 := uint32(numElts) - (*funcs)[index] = func(p *wrappers.Packer) error { - p.PackInt(numEltsAsUint32) // pack # elements + p.PackInt(uint32(numElts)) // pack # elements + if p.Err != nil { return p.Err } - funcsWritten = subFuncsWritten + 1 - return + + for i := 0; i < numElts; i++ { // Process each element in the slice + if err := c.marshal(value.Index(i), p); err != nil { + return err + } + } + + return nil case reflect.Array: numElts := value.Len() if numElts > c.maxSliceLen { - return 0, 0, fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) + return fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } - size = 0 - funcsWritten = 0 for i := 0; i < numElts; i++ { // Process each element in the array - subSize, n, subErr := c.marshal(value.Index(i), index+funcsWritten, funcs) - if subErr != nil { - return 0, 0, subErr + if err := c.marshal(value.Index(i), p); err != nil { + return err } - size += subSize - funcsWritten += n } - return + return nil case reflect.Struct: serializedFields, subErr := c.getSerializedFieldIndices(value.Type()) if subErr != nil { - return 0, 0, subErr + return subErr } - size = 0 - funcsWritten = 0 for _, fieldIndex := range serializedFields { // Go through all fields of this struct - subSize, n, err := c.marshal(value.Field(fieldIndex), index+funcsWritten, funcs) // Serialize the field - if err != nil { - return 0, 0, err + if err := c.marshal(value.Field(fieldIndex), p); err != nil { // Serialize the field + return err } - size += subSize - funcsWritten += n } - return + return nil default: - return 0, 0, errUnknownType + return errUnknownType } } From ba2b214b587310f074dfb62bc3dfce054faa0fa5 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 12 Jun 2020 22:20:50 -0400 Subject: [PATCH 27/42] Decouple staking and tls encryption for internode communication --- main/main.go | 5 ++++- main/params.go | 15 ++++++++++++--- node/config.go | 1 + node/node.go | 4 ++-- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/main/main.go b/main/main.go index 5aca025..98cb581 100644 --- a/main/main.go +++ b/main/main.go @@ -45,7 +45,10 @@ func main() { } // Track if sybil control is enforced - if !Config.EnableStaking { + if !Config.EnableStaking && Config.EnableP2PTLS { + log.Warn("Staking is disabled. Sybil control is not enforced.") + } + if !Config.EnableStaking && !Config.EnableP2PTLS { log.Warn("Staking and p2p encryption are disabled. Packet spoofing is possible.") } diff --git a/main/params.go b/main/params.go index eef8e60..150c0e8 100644 --- a/main/params.go +++ b/main/params.go @@ -49,7 +49,8 @@ var ( ) var ( - errBootstrapMismatch = errors.New("more bootstrap IDs provided than bootstrap IPs") + errBootstrapMismatch = errors.New("more bootstrap IDs provided than bootstrap IPs") + errStakingRequiresTLS = errors.New("if staking is enabled, network TLS must also be enabled") ) // GetIPs returns the default IPs for each network @@ -200,7 +201,9 @@ func init() { // Staking: consensusPort := fs.Uint("staking-port", 9651, "Port of the consensus server") - fs.BoolVar(&Config.EnableStaking, "staking-tls-enabled", true, "Require TLS to authenticate staking connections") + // TODO - keeping same flag for backwards compatibility, should be changed to "staking-enabled" + fs.BoolVar(&Config.EnableStaking, "staking-tls-enabled", true, "Enable staking. If enabled, Network TLS is required.") + fs.BoolVar(&Config.EnableP2PTLS, "p2p-tls-enabled", true, "Require TLS to authenticate network communication") fs.StringVar(&Config.StakingKeyFile, "staking-tls-key-file", defaultStakingKeyPath, "TLS private key for staking") fs.StringVar(&Config.StakingCertFile, "staking-tls-cert-file", defaultStakingCertPath, "TLS certificate for staking") @@ -318,7 +321,13 @@ func init() { *bootstrapIDs = strings.Join(defaultBootstrapIDs, ",") } } - if Config.EnableStaking { + + if Config.EnableStaking && !Config.EnableP2PTLS { + errs.Add(errStakingRequiresTLS) + return + } + + if Config.EnableP2PTLS { i := 0 cb58 := formatting.CB58{} for _, id := range strings.Split(*bootstrapIDs, ",") { diff --git a/node/config.go b/node/config.go index 74ff491..2504276 100644 --- a/node/config.go +++ b/node/config.go @@ -34,6 +34,7 @@ type Config struct { // Staking configuration StakingIP utils.IPDesc + EnableP2PTLS bool EnableStaking bool StakingKeyFile string StakingCertFile string diff --git a/node/node.go b/node/node.go index ea0e8fc..5e817fa 100644 --- a/node/node.go +++ b/node/node.go @@ -119,7 +119,7 @@ func (n *Node) initNetworking() error { dialer := network.NewDialer(TCP) var serverUpgrader, clientUpgrader network.Upgrader - if n.Config.EnableStaking { + if n.Config.EnableP2PTLS { cert, err := tls.LoadX509KeyPair(n.Config.StakingCertFile, n.Config.StakingKeyFile) if err != nil { return err @@ -253,7 +253,7 @@ func (n *Node) initDatabase() error { // Otherwise, it is a hash of the TLS certificate that this node // uses for P2P communication func (n *Node) initNodeID() error { - if !n.Config.EnableStaking { + if !n.Config.EnableP2PTLS { n.ID = ids.NewShortID(hashing.ComputeHash160Array([]byte(n.Config.StakingIP.String()))) n.Log.Info("Set the node's ID to %s", n.ID) return nil From f28b69b81940ca08d0567bad854cc73db7fffabb Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 10:06:40 -0400 Subject: [PATCH 28/42] set initial slice capacity for packer; packer pre-allocates capacity when possible --- utils/wrappers/packing.go | 35 ++++--- vms/components/codec/codec.go | 167 +++++++++++++++++++--------------- 2 files changed, 118 insertions(+), 84 deletions(-) diff --git a/utils/wrappers/packing.go b/utils/wrappers/packing.go index 1038852..22c7464 100644 --- a/utils/wrappers/packing.go +++ b/utils/wrappers/packing.go @@ -16,6 +16,11 @@ const ( // MaxStringLen ... MaxStringLen = math.MaxUint16 + // When the byte array is expanded, this many extra bytes + // are added to capacity of the array. + // Higher value --> need to expand byte array less --> less memory allocations + expansionBoost = 256 + // ByteLen is the number of bytes per byte... ByteLen = 1 // ShortLen is the number of bytes per short @@ -61,27 +66,35 @@ func (p *Packer) CheckSpace(bytes int) { } } -// Expand ensures that there is [bytes] bytes left of space in the byte array. -// If this is not allowed due to the maximum size, an error is added to the -// packer +// Expand ensures that there is [bytes] bytes left of space in the byte slice. +// If this is not allowed due to the maximum size, an error is added to the packer +// In order to understand this code, its important to understand the difference +// between a slice's length and its capacity. func (p *Packer) Expand(bytes int) { p.CheckSpace(0) if p.Errored() { return } - neededSize := bytes + p.Offset - if neededSize <= len(p.Bytes) { + neededSize := bytes + p.Offset // Need byte slice's length to be at least [neededSize] + if neededSize <= len(p.Bytes) { // Byte slice has sufficient length already + return + } else if neededSize > p.MaxSize { // Lengthening the byte slice would cause it to grow too large + p.Add(errBadLength) + return + } else if neededSize <= cap(p.Bytes) { // Byte slice has sufficient capacity to lengthen it without mem alloc + p.Bytes = p.Bytes[:neededSize] return } - if neededSize > p.MaxSize { - p.Add(errBadLength) - } else if neededSize > cap(p.Bytes) { - p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes))...) - } else { - p.Bytes = p.Bytes[:neededSize] + // See if we can expand the byte slice an extra [expansionBoost] bytes in order to + // prevent need for future expansions (and therefore memory allocations) + capToAdd := neededSize - cap(p.Bytes) + expansionBoost + if capToAdd > p.MaxSize { + capToAdd = neededSize - cap(p.Bytes) } + // increase slice's length and capacity + p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes), capToAdd)...) } // PackByte append a byte to the byte array diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 8e223f9..2f974a9 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -6,7 +6,6 @@ package codec import ( "errors" "fmt" - "math" "reflect" "unicode" @@ -16,22 +15,16 @@ import ( const ( defaultMaxSize = 1 << 18 // default max size, in bytes, of something being marshalled by Marshal() defaultMaxSliceLength = 1 << 18 // default max length of a slice being marshalled by Marshal(). Should be <= math.MaxUint32. - maxStringLen = math.MaxUint16 + + // initial capacity of byte slice that values are marshaled into. + // Larger value --> need less memory allocations but possibly have allocated but unused memory + // Smaller value --> need more memory allocations but more efficient use of allocated memory + initialSliceCap = 2048 ) -// ErrBadCodec is returned when one tries to perform an operation -// using an unknown codec var ( - errBadCodec = errors.New("wrong or unknown codec used") - errNil = errors.New("can't marshal/unmarshal nil value") - errNeedPointer = errors.New("must unmarshal into a pointer") - errMarshalUnregisteredType = errors.New("can't marshal an unregistered type") - errUnmarshalUnregisteredType = errors.New("can't unmarshal an unregistered type") - errUnknownType = errors.New("don't know how to marshal/unmarshal this type") - errMarshalUnexportedField = errors.New("can't serialize an unexported field") - errUnmarshalUnexportedField = errors.New("can't deserialize into an unexported field") - errOutOfMemory = errors.New("out of memory") - errSliceTooLarge = errors.New("slice too large") + errNil = errors.New("can't marshal/unmarshal nil pointer or interface") + errNeedPointer = errors.New("argument to unmarshal should be a pointer") ) // Codec handles marshaling and unmarshaling of structs @@ -42,6 +35,12 @@ type codec struct { typeIDToType map[uint32]reflect.Type typeToTypeID map[reflect.Type]uint32 + // Key: a struct type + // Value: Slice where each element is index in the struct type + // of a field that is serialized/deserialized + // e.g. Foo --> [1,5,8] means Foo.Field(1), etc. are to be serialized/deserialized + // We assume this cache is pretty small (a few hundred keys at most) + // and doesn't take up much memory serializedFieldIndices map[reflect.Type][]int } @@ -94,10 +93,10 @@ func (c *codec) RegisterType(val interface{}) error { // To marshal an interface, [value] must be a pointer to the interface func (c *codec) Marshal(value interface{}) ([]byte, error) { if value == nil { - return nil, errNil + return nil, errNil // can't marshal nil } - p := &wrappers.Packer{MaxSize: 512, Bytes: make([]byte, 0, 512)} + p := &wrappers.Packer{MaxSize: c.maxSize, Bytes: make([]byte, 0, initialSliceCap)} if err := c.marshal(reflect.ValueOf(value), p); err != nil { return nil, err } @@ -105,16 +104,10 @@ func (c *codec) Marshal(value interface{}) ([]byte, error) { return p.Bytes, nil } -// marshal returns: -// 1) The size, in bytes, of the byte representation of [value] -// 2) A slice of functions, where each function writes bytes to its argument -// and returns the number of bytes it wrote. -// When these functions are called in order, they write [value] to a byte slice. -// 3) An error +// marshal writes the byte representation of [value] to [p] +// [value]'s underlying value must not be a nil pointer or interface func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { valueKind := value.Kind() - - // Case: Value can't be marshalled switch valueKind { case reflect.Interface, reflect.Ptr, reflect.Invalid: if value.IsNil() { // Can't marshal nil (except nil slices) @@ -161,7 +154,7 @@ func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { if !ok { return fmt.Errorf("can't marshal unregistered type '%v'", reflect.TypeOf(underlyingValue).String()) } - p.PackInt(typeID) + p.PackInt(typeID) // Pack type ID so we know what to unmarshal this into if p.Err != nil { return p.Err } @@ -178,20 +171,17 @@ func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { if p.Err != nil { return p.Err } - for i := 0; i < numElts; i++ { // Process each element in the slice if err := c.marshal(value.Index(i), p); err != nil { return err } } - return nil case reflect.Array: numElts := value.Len() if numElts > c.maxSliceLen { return fmt.Errorf("array length, %d, exceeds maximum length, %d", numElts, c.maxSliceLen) } - for i := 0; i < numElts; i++ { // Process each element in the array if err := c.marshal(value.Index(i), p); err != nil { return err @@ -199,19 +189,18 @@ func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { } return nil case reflect.Struct: - serializedFields, subErr := c.getSerializedFieldIndices(value.Type()) - if subErr != nil { - return subErr + serializedFields, err := c.getSerializedFieldIndices(value.Type()) + if err != nil { + return err } - - for _, fieldIndex := range serializedFields { // Go through all fields of this struct - if err := c.marshal(value.Field(fieldIndex), p); err != nil { // Serialize the field + for _, fieldIndex := range serializedFields { // Go through all fields of this struct that are serialized + if err := c.marshal(value.Field(fieldIndex), p); err != nil { // Serialize the field and write to byte array return err } } return nil default: - return errUnknownType + return fmt.Errorf("can't marshal unknown kind %s", valueKind) } } @@ -220,7 +209,7 @@ func (c *codec) marshal(value reflect.Value, p *wrappers.Packer) error { func (c *codec) Unmarshal(bytes []byte, dest interface{}) error { switch { case len(bytes) > c.maxSize: - return errSliceTooLarge + return fmt.Errorf("byte array exceeds maximum length, %d", c.maxSize) case dest == nil: return errNil } @@ -239,121 +228,153 @@ func (c *codec) Unmarshal(bytes []byte, dest interface{}) error { return nil } -// Unmarshal from [bytes] into [value]. [value] must be addressable +// Unmarshal from p.Bytes into [value]. [value] must be addressable. func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { switch value.Kind() { case reflect.Uint8: value.SetUint(uint64(p.UnpackByte())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal uint8: %s", p.Err) + } + return nil case reflect.Int8: value.SetInt(int64(p.UnpackByte())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal int8: %s", p.Err) + } + return nil case reflect.Uint16: value.SetUint(uint64(p.UnpackShort())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal uint16: %s", p.Err) + } + return nil case reflect.Int16: value.SetInt(int64(p.UnpackShort())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal int16: %s", p.Err) + } + return nil case reflect.Uint32: value.SetUint(uint64(p.UnpackInt())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal uint32: %s", p.Err) + } + return nil case reflect.Int32: value.SetInt(int64(p.UnpackInt())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal int32: %s", p.Err) + } + return nil case reflect.Uint64: value.SetUint(uint64(p.UnpackLong())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal uint64: %s", p.Err) + } + return nil case reflect.Int64: value.SetInt(int64(p.UnpackLong())) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal int64: %s", p.Err) + } + return nil case reflect.Bool: value.SetBool(p.UnpackBool()) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal bool: %s", p.Err) + } + return nil case reflect.Slice: numElts := int(p.UnpackInt()) if p.Err != nil { - return p.Err + return fmt.Errorf("couldn't marshal slice: %s", p.Err) } - // set [value] to be a slice of the appropriate type/capacity (right now [value] is nil) + // set [value] to be a slice of the appropriate type/capacity (right now it is nil) value.Set(reflect.MakeSlice(value.Type(), numElts, numElts)) // Unmarshal each element into the appropriate index of the slice for i := 0; i < numElts; i++ { if err := c.unmarshal(p, value.Index(i)); err != nil { - return err + return fmt.Errorf("couldn't marshal slice element: %s", err) } } return nil case reflect.Array: for i := 0; i < value.Len(); i++ { if err := c.unmarshal(p, value.Index(i)); err != nil { - return err + return fmt.Errorf("couldn't marshal array element: %s", err) } } return nil case reflect.String: value.SetString(p.UnpackStr()) - return p.Err + if p.Err != nil { + return fmt.Errorf("couldn't marshal string: %s", p.Err) + } + return nil case reflect.Interface: typeID := p.UnpackInt() // Get the type ID if p.Err != nil { - return p.Err + return fmt.Errorf("couldn't marshal interface: %s", p.Err) } // Get a type that implements the interface - typ, ok := c.typeIDToType[typeID] + implementingType, ok := c.typeIDToType[typeID] if !ok { - return errUnmarshalUnregisteredType + return fmt.Errorf("couldn't marshal interface: unknown type ID %d", typeID) } // Ensure type actually does implement the interface - if valueType := value.Type(); !typ.Implements(valueType) { - return fmt.Errorf("%s does not implement interface %s", typ, valueType) + if valueType := value.Type(); !implementingType.Implements(valueType) { + return fmt.Errorf("couldn't marshal interface: %s does not implement interface %s", implementingType, valueType) } - concreteInstance := reflect.New(typ).Elem() // instance of the proper type + intfImplementor := reflect.New(implementingType).Elem() // instance of the proper type // Unmarshal into the struct - if err := c.unmarshal(p, concreteInstance); err != nil { - return err + if err := c.unmarshal(p, intfImplementor); err != nil { + return fmt.Errorf("couldn't marshal interface: %s", err) } // And assign the filled struct to the value - value.Set(concreteInstance) + value.Set(intfImplementor) return nil case reflect.Struct: - // Type of this struct + // Get indices of fields that will be unmarshaled into serializedFieldIndices, err := c.getSerializedFieldIndices(value.Type()) if err != nil { - return err + return fmt.Errorf("couldn't marshal struct: %s", err) } - // Go through all the fields and umarshal into each + // Go through the fields and umarshal into them for _, index := range serializedFieldIndices { - if err := c.unmarshal(p, value.Field(index)); err != nil { // Unmarshal into the field - return err + if err := c.unmarshal(p, value.Field(index)); err != nil { + return fmt.Errorf("couldn't marshal struct: %s", err) } } return nil case reflect.Ptr: // Get the type this pointer points to - underlyingType := value.Type().Elem() + t := value.Type().Elem() // Create a new pointer to a new value of the underlying type - underlyingValue := reflect.New(underlyingType) + v := reflect.New(t) // Fill the value - if err := c.unmarshal(p, underlyingValue.Elem()); err != nil { - return err + if err := c.unmarshal(p, v.Elem()); err != nil { + return fmt.Errorf("couldn't marshal pointer: %s", err) } // Assign to the top-level struct's member - value.Set(underlyingValue) + value.Set(v) return nil case reflect.Invalid: return errNil default: - return errUnknownType + return fmt.Errorf("can't unmarshal unknown type %s", value.Kind().String()) } } // Returns the indices of the serializable fields of [t], which is a struct type // Returns an error if a field has tag "serialize: true" but the field is unexported +// e.g. getSerializedFieldIndices(Foo) --> [1,5,8] means Foo.Field(1), Foo.Field(5), Foo.Field(8) +// are to be serialized/deserialized func (c *codec) getSerializedFieldIndices(t reflect.Type) ([]int, error) { if c.serializedFieldIndices == nil { c.serializedFieldIndices = make(map[reflect.Type][]int) } - if serializedFields, ok := c.serializedFieldIndices[t]; ok { + if serializedFields, ok := c.serializedFieldIndices[t]; ok { // use pre-computed result return serializedFields, nil } numFields := t.NumField() @@ -368,6 +389,6 @@ func (c *codec) getSerializedFieldIndices(t reflect.Type) ([]int, error) { } serializedFields = append(serializedFields, i) } - c.serializedFieldIndices[t] = serializedFields + c.serializedFieldIndices[t] = serializedFields // cache result return serializedFields, nil } From cea79f66b29739ec5934232a2bc4cdd7f60571cd Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 10:33:08 -0400 Subject: [PATCH 29/42] add to tests; comment them --- vms/components/codec/codec_test.go | 82 +++++++++++++++++++----------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/vms/components/codec/codec_test.go b/vms/components/codec/codec_test.go index 74209eb..edd3f85 100644 --- a/vms/components/codec/codec_test.go +++ b/vms/components/codec/codec_test.go @@ -5,6 +5,7 @@ package codec import ( "bytes" + "math" "reflect" "testing" ) @@ -104,36 +105,8 @@ func TestStruct(t *testing.T) { t.Fatal(err) } - if !reflect.DeepEqual(myStructUnmarshaled.Member1, myStructInstance.Member1) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !bytes.Equal(myStructUnmarshaled.MySlice, myStructInstance.MySlice) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MySlice2, myStructInstance.MySlice2) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MySlice3, myStructInstance.MySlice3) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MySlice3, myStructInstance.MySlice3) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MySlice4, myStructInstance.MySlice4) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.InnerStruct, myStructInstance.InnerStruct) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.InnerStruct2, myStructInstance.InnerStruct2) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MyArray2, myStructInstance.MyArray2) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MyArray3, myStructInstance.MyArray3) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MyArray4, myStructInstance.MyArray4) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MyInterface, myStructInstance.MyInterface) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MySlice5, myStructInstance.MySlice5) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.InnerStruct3, myStructInstance.InnerStruct3) { - t.Fatal("expected unmarshaled struct to be same as original struct") - } else if !reflect.DeepEqual(myStructUnmarshaled.MyPointer, myStructInstance.MyPointer) { - t.Fatal("expected unmarshaled struct to be same as original struct") + if !reflect.DeepEqual(*myStructUnmarshaled, myStructInstance) { + t.Fatal("should be same") } } @@ -173,6 +146,28 @@ func TestSlice(t *testing.T) { } } +// Test marshalling/unmarshalling largest possible slice +func TestMaxSizeSlice(t *testing.T) { + mySlice := make([]string, math.MaxUint16, math.MaxUint16) + mySlice[0] = "first!" + mySlice[math.MaxUint16-1] = "last!" + codec := NewDefault() + bytes, err := codec.Marshal(mySlice) + if err != nil { + t.Fatal(err) + } + + var sliceUnmarshaled []string + if err := codec.Unmarshal(bytes, &sliceUnmarshaled); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(mySlice, sliceUnmarshaled) { + t.Fatal("expected marshaled and unmarshaled values to match") + } +} + +// Test marshalling a bool func TestBool(t *testing.T) { myBool := true codec := NewDefault() @@ -191,6 +186,7 @@ func TestBool(t *testing.T) { } } +// Test marshalling an array func TestArray(t *testing.T) { myArr := [5]uint64{5, 6, 7, 8, 9} codec := NewDefault() @@ -209,6 +205,26 @@ func TestArray(t *testing.T) { } } +// Test marshalling a really big array +func TestBigArray(t *testing.T) { + myArr := [30000]uint64{5, 6, 7, 8, 9} + codec := NewDefault() + bytes, err := codec.Marshal(myArr) + if err != nil { + t.Fatal(err) + } + + var myArrUnmarshaled [30000]uint64 + if err := codec.Unmarshal(bytes, &myArrUnmarshaled); err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(myArr, myArrUnmarshaled) { + t.Fatal("expected marshaled and unmarshaled values to match") + } +} + +// Test marshalling a pointer to a struct func TestPointerToStruct(t *testing.T) { myPtr := &MyInnerStruct{Str: "Hello!"} codec := NewDefault() @@ -227,6 +243,7 @@ func TestPointerToStruct(t *testing.T) { } } +// Test marshalling a slice of structs func TestSliceOfStruct(t *testing.T) { mySlice := []MyInnerStruct3{ MyInnerStruct3{ @@ -257,6 +274,7 @@ func TestSliceOfStruct(t *testing.T) { } } +// Test marshalling an interface func TestInterface(t *testing.T) { codec := NewDefault() codec.RegisterType(&MyInnerStruct2{}) @@ -278,6 +296,7 @@ func TestInterface(t *testing.T) { } } +// Test marshalling a slice of interfaces func TestSliceOfInterface(t *testing.T) { mySlice := []Foo{ &MyInnerStruct{ @@ -304,6 +323,7 @@ func TestSliceOfInterface(t *testing.T) { } } +// Test marshalling an array of interfaces func TestArrayOfInterface(t *testing.T) { myArray := [2]Foo{ &MyInnerStruct{ @@ -330,6 +350,7 @@ func TestArrayOfInterface(t *testing.T) { } } +// Test marshalling a pointer to an interface func TestPointerToInterface(t *testing.T) { var myinnerStruct Foo = &MyInnerStruct{Str: "Hello!"} var myPtr *Foo = &myinnerStruct @@ -352,6 +373,7 @@ func TestPointerToInterface(t *testing.T) { } } +// Test marshalling a string func TestString(t *testing.T) { myString := "Ayy" codec := NewDefault() From 979477d68f4102df8c3fe0bcbfdd6d3a4d758b36 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 10:46:22 -0400 Subject: [PATCH 30/42] change initialSliceCap --> 1024 --- vms/components/codec/codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 2f974a9..d644ad5 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -19,7 +19,7 @@ const ( // initial capacity of byte slice that values are marshaled into. // Larger value --> need less memory allocations but possibly have allocated but unused memory // Smaller value --> need more memory allocations but more efficient use of allocated memory - initialSliceCap = 2048 + initialSliceCap = 1024 ) var ( From a84abacea5fa07ae44276f6bccc30b34673816f1 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 10:55:09 -0400 Subject: [PATCH 31/42] fix typos and removed useless benchmark --- vms/components/codec/codec.go | 40 ++++++++++---------- vms/components/codec/codec_benchmark_test.go | 9 ----- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index d644ad5..53852a9 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -234,102 +234,102 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { case reflect.Uint8: value.SetUint(uint64(p.UnpackByte())) if p.Err != nil { - return fmt.Errorf("couldn't marshal uint8: %s", p.Err) + return fmt.Errorf("couldn't unmarshal uint8: %s", p.Err) } return nil case reflect.Int8: value.SetInt(int64(p.UnpackByte())) if p.Err != nil { - return fmt.Errorf("couldn't marshal int8: %s", p.Err) + return fmt.Errorf("couldn't unmarshal int8: %s", p.Err) } return nil case reflect.Uint16: value.SetUint(uint64(p.UnpackShort())) if p.Err != nil { - return fmt.Errorf("couldn't marshal uint16: %s", p.Err) + return fmt.Errorf("couldn't unmarshal uint16: %s", p.Err) } return nil case reflect.Int16: value.SetInt(int64(p.UnpackShort())) if p.Err != nil { - return fmt.Errorf("couldn't marshal int16: %s", p.Err) + return fmt.Errorf("couldn't unmarshal int16: %s", p.Err) } return nil case reflect.Uint32: value.SetUint(uint64(p.UnpackInt())) if p.Err != nil { - return fmt.Errorf("couldn't marshal uint32: %s", p.Err) + return fmt.Errorf("couldn't unmarshal uint32: %s", p.Err) } return nil case reflect.Int32: value.SetInt(int64(p.UnpackInt())) if p.Err != nil { - return fmt.Errorf("couldn't marshal int32: %s", p.Err) + return fmt.Errorf("couldn't unmarshal int32: %s", p.Err) } return nil case reflect.Uint64: value.SetUint(uint64(p.UnpackLong())) if p.Err != nil { - return fmt.Errorf("couldn't marshal uint64: %s", p.Err) + return fmt.Errorf("couldn't unmarshal uint64: %s", p.Err) } return nil case reflect.Int64: value.SetInt(int64(p.UnpackLong())) if p.Err != nil { - return fmt.Errorf("couldn't marshal int64: %s", p.Err) + return fmt.Errorf("couldn't unmarshal int64: %s", p.Err) } return nil case reflect.Bool: value.SetBool(p.UnpackBool()) if p.Err != nil { - return fmt.Errorf("couldn't marshal bool: %s", p.Err) + return fmt.Errorf("couldn't unmarshal bool: %s", p.Err) } return nil case reflect.Slice: numElts := int(p.UnpackInt()) if p.Err != nil { - return fmt.Errorf("couldn't marshal slice: %s", p.Err) + return fmt.Errorf("couldn't unmarshal slice: %s", p.Err) } // set [value] to be a slice of the appropriate type/capacity (right now it is nil) value.Set(reflect.MakeSlice(value.Type(), numElts, numElts)) // Unmarshal each element into the appropriate index of the slice for i := 0; i < numElts; i++ { if err := c.unmarshal(p, value.Index(i)); err != nil { - return fmt.Errorf("couldn't marshal slice element: %s", err) + return fmt.Errorf("couldn't unmarshal slice element: %s", err) } } return nil case reflect.Array: for i := 0; i < value.Len(); i++ { if err := c.unmarshal(p, value.Index(i)); err != nil { - return fmt.Errorf("couldn't marshal array element: %s", err) + return fmt.Errorf("couldn't unmarshal array element: %s", err) } } return nil case reflect.String: value.SetString(p.UnpackStr()) if p.Err != nil { - return fmt.Errorf("couldn't marshal string: %s", p.Err) + return fmt.Errorf("couldn't unmarshal string: %s", p.Err) } return nil case reflect.Interface: typeID := p.UnpackInt() // Get the type ID if p.Err != nil { - return fmt.Errorf("couldn't marshal interface: %s", p.Err) + return fmt.Errorf("couldn't unmarshal interface: %s", p.Err) } // Get a type that implements the interface implementingType, ok := c.typeIDToType[typeID] if !ok { - return fmt.Errorf("couldn't marshal interface: unknown type ID %d", typeID) + return fmt.Errorf("couldn't unmarshal interface: unknown type ID %d", typeID) } // Ensure type actually does implement the interface if valueType := value.Type(); !implementingType.Implements(valueType) { - return fmt.Errorf("couldn't marshal interface: %s does not implement interface %s", implementingType, valueType) + return fmt.Errorf("couldn't unmarshal interface: %s does not implement interface %s", implementingType, valueType) } intfImplementor := reflect.New(implementingType).Elem() // instance of the proper type // Unmarshal into the struct if err := c.unmarshal(p, intfImplementor); err != nil { - return fmt.Errorf("couldn't marshal interface: %s", err) + return fmt.Errorf("couldn't unmarshal interface: %s", err) } // And assign the filled struct to the value value.Set(intfImplementor) @@ -338,12 +338,12 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { // Get indices of fields that will be unmarshaled into serializedFieldIndices, err := c.getSerializedFieldIndices(value.Type()) if err != nil { - return fmt.Errorf("couldn't marshal struct: %s", err) + return fmt.Errorf("couldn't unmarshal struct: %s", err) } // Go through the fields and umarshal into them for _, index := range serializedFieldIndices { if err := c.unmarshal(p, value.Field(index)); err != nil { - return fmt.Errorf("couldn't marshal struct: %s", err) + return fmt.Errorf("couldn't unmarshal struct: %s", err) } } return nil @@ -354,7 +354,7 @@ func (c *codec) unmarshal(p *wrappers.Packer, value reflect.Value) error { v := reflect.New(t) // Fill the value if err := c.unmarshal(p, v.Elem()); err != nil { - return fmt.Errorf("couldn't marshal pointer: %s", err) + return fmt.Errorf("couldn't unmarshal pointer: %s", err) } // Assign to the top-level struct's member value.Set(v) diff --git a/vms/components/codec/codec_benchmark_test.go b/vms/components/codec/codec_benchmark_test.go index 25af563..4adfa52 100644 --- a/vms/components/codec/codec_benchmark_test.go +++ b/vms/components/codec/codec_benchmark_test.go @@ -62,12 +62,3 @@ func BenchmarkMarshalNonCodec(b *testing.B) { } } } - -func BenchmarkFoo(b *testing.B) { - arr := make([]int, 10000, 10000) - for n := 0; n < b.N; n++ { - for i := 0; i < 10000; i++ { - arr[i] = i - } - } -} From f59f45a20fb8eaee395e2a2e269847e79d43bad0 Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Mon, 15 Jun 2020 09:35:41 -0700 Subject: [PATCH 32/42] Make all RPC call logs `Info` level. --- api/admin/service.go | 24 ++++++++++---------- api/health/service.go | 2 +- api/ipcs/server.go | 4 ++-- api/keystore/service.go | 10 ++++----- vms/avm/service.go | 34 ++++++++++++++--------------- vms/platformvm/service.go | 46 +++++++++++++++++++-------------------- 6 files changed, 60 insertions(+), 60 deletions(-) diff --git a/api/admin/service.go b/api/admin/service.go index e05a440..0718dfd 100644 --- a/api/admin/service.go +++ b/api/admin/service.go @@ -57,7 +57,7 @@ type GetNodeVersionReply struct { // GetNodeVersion returns the version this node is running func (service *Admin) GetNodeVersion(_ *http.Request, _ *struct{}, reply *GetNodeVersionReply) error { - service.log.Debug("Admin: GetNodeVersion called") + service.log.Info("Admin: GetNodeVersion called") reply.Version = service.version.String() return nil @@ -70,7 +70,7 @@ type GetNodeIDReply struct { // GetNodeID returns the node ID of this node func (service *Admin) GetNodeID(_ *http.Request, _ *struct{}, reply *GetNodeIDReply) error { - service.log.Debug("Admin: GetNodeID called") + service.log.Info("Admin: GetNodeID called") reply.NodeID = service.nodeID return nil @@ -83,7 +83,7 @@ type GetNetworkIDReply struct { // GetNetworkID returns the network ID this node is running on func (service *Admin) GetNetworkID(_ *http.Request, _ *struct{}, reply *GetNetworkIDReply) error { - service.log.Debug("Admin: GetNetworkID called") + service.log.Info("Admin: GetNetworkID called") reply.NetworkID = cjson.Uint32(service.networkID) return nil @@ -96,7 +96,7 @@ type GetNetworkNameReply struct { // GetNetworkName returns the network name this node is running on func (service *Admin) GetNetworkName(_ *http.Request, _ *struct{}, reply *GetNetworkNameReply) error { - service.log.Debug("Admin: GetNetworkName called") + service.log.Info("Admin: GetNetworkName called") reply.NetworkName = genesis.NetworkName(service.networkID) return nil @@ -114,7 +114,7 @@ type GetBlockchainIDReply struct { // GetBlockchainID returns the blockchain ID that resolves the alias that was supplied func (service *Admin) GetBlockchainID(_ *http.Request, args *GetBlockchainIDArgs, reply *GetBlockchainIDReply) error { - service.log.Debug("Admin: GetBlockchainID called") + service.log.Info("Admin: GetBlockchainID called") bID, err := service.chainManager.Lookup(args.Alias) reply.BlockchainID = bID.String() @@ -128,7 +128,7 @@ type PeersReply struct { // Peers returns the list of current validators func (service *Admin) Peers(_ *http.Request, _ *struct{}, reply *PeersReply) error { - service.log.Debug("Admin: Peers called") + service.log.Info("Admin: Peers called") reply.Peers = service.networking.Peers() return nil } @@ -145,7 +145,7 @@ type StartCPUProfilerReply struct { // StartCPUProfiler starts a cpu profile writing to the specified file func (service *Admin) StartCPUProfiler(_ *http.Request, args *StartCPUProfilerArgs, reply *StartCPUProfilerReply) error { - service.log.Debug("Admin: StartCPUProfiler called with %s", args.Filename) + service.log.Info("Admin: StartCPUProfiler called with %s", args.Filename) reply.Success = true return service.performance.StartCPUProfiler(args.Filename) } @@ -157,7 +157,7 @@ type StopCPUProfilerReply struct { // StopCPUProfiler stops the cpu profile func (service *Admin) StopCPUProfiler(_ *http.Request, _ *struct{}, reply *StopCPUProfilerReply) error { - service.log.Debug("Admin: StopCPUProfiler called") + service.log.Info("Admin: StopCPUProfiler called") reply.Success = true return service.performance.StopCPUProfiler() } @@ -174,7 +174,7 @@ type MemoryProfileReply struct { // MemoryProfile runs a memory profile writing to the specified file func (service *Admin) MemoryProfile(_ *http.Request, args *MemoryProfileArgs, reply *MemoryProfileReply) error { - service.log.Debug("Admin: MemoryProfile called with %s", args.Filename) + service.log.Info("Admin: MemoryProfile called with %s", args.Filename) reply.Success = true return service.performance.MemoryProfile(args.Filename) } @@ -191,7 +191,7 @@ type LockProfileReply struct { // LockProfile runs a mutex profile writing to the specified file func (service *Admin) LockProfile(_ *http.Request, args *LockProfileArgs, reply *LockProfileReply) error { - service.log.Debug("Admin: LockProfile called with %s", args.Filename) + service.log.Info("Admin: LockProfile called with %s", args.Filename) reply.Success = true return service.performance.LockProfile(args.Filename) } @@ -209,7 +209,7 @@ type AliasReply struct { // Alias attempts to alias an HTTP endpoint to a new name func (service *Admin) Alias(_ *http.Request, args *AliasArgs, reply *AliasReply) error { - service.log.Debug("Admin: Alias called with URL: %s, Alias: %s", args.Endpoint, args.Alias) + service.log.Info("Admin: Alias called with URL: %s, Alias: %s", args.Endpoint, args.Alias) reply.Success = true return service.httpServer.AddAliasesWithReadLock(args.Endpoint, args.Alias) } @@ -227,7 +227,7 @@ type AliasChainReply struct { // AliasChain attempts to alias a chain to a new name func (service *Admin) AliasChain(_ *http.Request, args *AliasChainArgs, reply *AliasChainReply) error { - service.log.Debug("Admin: AliasChain called with Chain: %s, Alias: %s", args.Chain, args.Alias) + service.log.Info("Admin: AliasChain called with Chain: %s, Alias: %s", args.Chain, args.Alias) chainID, err := service.chainManager.Lookup(args.Chain) if err != nil { diff --git a/api/health/service.go b/api/health/service.go index db33640..fdd405b 100644 --- a/api/health/service.go +++ b/api/health/service.go @@ -74,7 +74,7 @@ type GetLivenessReply struct { // GetLiveness returns a summation of the health of the node func (h *Health) GetLiveness(_ *http.Request, _ *GetLivenessArgs, reply *GetLivenessReply) error { - h.log.Debug("Health: GetLiveness called") + h.log.Info("Health: GetLiveness called") reply.Checks, reply.Healthy = h.health.Results() return nil } diff --git a/api/ipcs/server.go b/api/ipcs/server.go index f4be11b..72a78c1 100644 --- a/api/ipcs/server.go +++ b/api/ipcs/server.go @@ -61,7 +61,7 @@ type PublishBlockchainReply struct { // PublishBlockchain publishes the finalized accepted transactions from the blockchainID over the IPC func (ipc *IPCs) PublishBlockchain(r *http.Request, args *PublishBlockchainArgs, reply *PublishBlockchainReply) error { - ipc.log.Debug("IPCs: PublishBlockchain called with BlockchainID: %s", args.BlockchainID) + ipc.log.Info("IPCs: PublishBlockchain called with BlockchainID: %s", args.BlockchainID) chainID, err := ipc.chainManager.Lookup(args.BlockchainID) if err != nil { ipc.log.Error("unknown blockchainID: %s", err) @@ -117,7 +117,7 @@ type UnpublishBlockchainReply struct { // UnpublishBlockchain closes publishing of a blockchainID func (ipc *IPCs) UnpublishBlockchain(r *http.Request, args *UnpublishBlockchainArgs, reply *UnpublishBlockchainReply) error { - ipc.log.Debug("IPCs: UnpublishBlockchain called with BlockchainID: %s", args.BlockchainID) + ipc.log.Info("IPCs: UnpublishBlockchain called with BlockchainID: %s", args.BlockchainID) chainID, err := ipc.chainManager.Lookup(args.BlockchainID) if err != nil { ipc.log.Error("unknown blockchainID %s: %s", args.BlockchainID, err) diff --git a/api/keystore/service.go b/api/keystore/service.go index 7006073..bf2e1fc 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -135,7 +135,7 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("Keystore: CreateUser called with %.*s", maxUserPassLen, args.Username) + ks.log.Info("Keystore: CreateUser called with %.*s", maxUserPassLen, args.Username) if len(args.Username) > maxUserPassLen || len(args.Password) > maxUserPassLen { return errUserPassMaxLength @@ -183,7 +183,7 @@ func (ks *Keystore) ListUsers(_ *http.Request, args *ListUsersArgs, reply *ListU ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("Keystore: ListUsers called") + ks.log.Info("Keystore: ListUsers called") reply.Users = []string{} @@ -211,7 +211,7 @@ func (ks *Keystore) ExportUser(_ *http.Request, args *ExportUserArgs, reply *Exp ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("Keystore: ExportUser called for %s", args.Username) + ks.log.Info("Keystore: ExportUser called for %s", args.Username) usr, err := ks.getUser(args.Username) if err != nil { @@ -264,7 +264,7 @@ func (ks *Keystore) ImportUser(r *http.Request, args *ImportUserArgs, reply *Imp ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("Keystore: ImportUser called for %s", args.Username) + ks.log.Info("Keystore: ImportUser called for %s", args.Username) if args.Username == "" { return errEmptyUsername @@ -324,7 +324,7 @@ func (ks *Keystore) DeleteUser(_ *http.Request, args *DeleteUserArgs, reply *Del ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("Keystore: DeleteUser called with %s", args.Username) + ks.log.Info("Keystore: DeleteUser called with %s", args.Username) if args.Username == "" { return errEmptyUsername diff --git a/vms/avm/service.go b/vms/avm/service.go index 4033be4..02f0216 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -56,7 +56,7 @@ type IssueTxReply struct { // IssueTx attempts to issue a transaction into consensus func (service *Service) IssueTx(r *http.Request, args *IssueTxArgs, reply *IssueTxReply) error { - service.vm.ctx.Log.Debug("AVM: IssueTx called with %s", args.Tx) + service.vm.ctx.log.Info("AVM: IssueTx called with %s", args.Tx) txID, err := service.vm.IssueTx(args.Tx.Bytes, nil) if err != nil { @@ -79,7 +79,7 @@ type GetTxStatusReply struct { // GetTxStatus returns the status of the specified transaction func (service *Service) GetTxStatus(r *http.Request, args *GetTxStatusArgs, reply *GetTxStatusReply) error { - service.vm.ctx.Log.Debug("AVM: GetTxStatus called with %s", args.TxID) + service.vm.ctx.log.Info("AVM: GetTxStatus called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -106,7 +106,7 @@ type GetTxReply struct { // GetTx returns the specified transaction func (service *Service) GetTx(r *http.Request, args *GetTxArgs, reply *GetTxReply) error { - service.vm.ctx.Log.Debug("AVM: GetTx called with %s", args.TxID) + service.vm.ctx.log.Info("AVM: GetTx called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -136,7 +136,7 @@ type GetUTXOsReply struct { // GetUTXOs creates an empty account with the name passed in func (service *Service) GetUTXOs(r *http.Request, args *GetUTXOsArgs, reply *GetUTXOsReply) error { - service.vm.ctx.Log.Debug("AVM: GetUTXOs called with %s", args.Addresses) + service.vm.ctx.log.Info("AVM: GetUTXOs called with %s", args.Addresses) addrSet := ids.Set{} for _, addr := range args.Addresses { @@ -178,7 +178,7 @@ type GetAssetDescriptionReply struct { // GetAssetDescription creates an empty account with the name passed in func (service *Service) GetAssetDescription(_ *http.Request, args *GetAssetDescriptionArgs, reply *GetAssetDescriptionReply) error { - service.vm.ctx.Log.Debug("AVM: GetAssetDescription called with %s", args.AssetID) + service.vm.ctx.log.Info("AVM: GetAssetDescription called with %s", args.AssetID) assetID, err := service.vm.Lookup(args.AssetID) if err != nil { @@ -222,7 +222,7 @@ type GetBalanceReply struct { // GetBalance returns the amount of an asset that an address at least partially owns func (service *Service) GetBalance(r *http.Request, args *GetBalanceArgs, reply *GetBalanceReply) error { - service.vm.ctx.Log.Debug("AVM: GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) + service.vm.ctx.log.Info("AVM: GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) address, err := service.vm.Parse(args.Address) if err != nil { @@ -287,7 +287,7 @@ type GetAllBalancesReply struct { // Note that balances include assets that the address only _partially_ owns // (ie is one of several addresses specified in a multi-sig) func (service *Service) GetAllBalances(r *http.Request, args *GetAllBalancesArgs, reply *GetAllBalancesReply) error { - service.vm.ctx.Log.Debug("AVM: GetAllBalances called with address: %s", args.Address) + service.vm.ctx.log.Info("AVM: GetAllBalances called with address: %s", args.Address) address, err := service.vm.Parse(args.Address) if err != nil { @@ -360,7 +360,7 @@ type CreateFixedCapAssetReply struct { // CreateFixedCapAsset returns ID of the newly created asset func (service *Service) CreateFixedCapAsset(r *http.Request, args *CreateFixedCapAssetArgs, reply *CreateFixedCapAssetReply) error { - service.vm.ctx.Log.Debug("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", + service.vm.ctx.log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", args.Name, args.Symbol, len(args.InitialHolders), @@ -445,7 +445,7 @@ type CreateVariableCapAssetReply struct { // CreateVariableCapAsset returns ID of the newly created asset func (service *Service) CreateVariableCapAsset(r *http.Request, args *CreateVariableCapAssetArgs, reply *CreateVariableCapAssetReply) error { - service.vm.ctx.Log.Debug("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", + service.vm.ctx.log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", args.Name, args.Symbol, len(args.MinterSets), @@ -523,7 +523,7 @@ type CreateAddressReply struct { // CreateAddress creates an address for the user [args.Username] func (service *Service) CreateAddress(r *http.Request, args *CreateAddressArgs, reply *CreateAddressReply) error { - service.vm.ctx.Log.Debug("AVM: CreateAddress called for user '%s'", args.Username) + service.vm.ctx.log.Info("AVM: CreateAddress called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -603,7 +603,7 @@ type ExportKeyReply struct { // ExportKey returns a private key from the provided user func (service *Service) ExportKey(r *http.Request, args *ExportKeyArgs, reply *ExportKeyReply) error { - service.vm.ctx.Log.Debug("AVM: ExportKey called for user '%s'", args.Username) + service.vm.ctx.log.Info("AVM: ExportKey called for user '%s'", args.Username) address, err := service.vm.Parse(args.Address) if err != nil { @@ -645,7 +645,7 @@ type ImportKeyReply struct { // ImportKey adds a private key to the provided user func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *ImportKeyReply) error { - service.vm.ctx.Log.Debug("AVM: ImportKey called for user '%s'", args.Username) + service.vm.ctx.log.Info("AVM: ImportKey called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -692,7 +692,7 @@ type SendReply struct { // Send returns the ID of the newly created transaction func (service *Service) Send(r *http.Request, args *SendArgs, reply *SendReply) error { - service.vm.ctx.Log.Debug("AVM: Send called with username: %s", args.Username) + service.vm.ctx.log.Info("AVM: Send called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount @@ -873,7 +873,7 @@ type CreateMintTxReply struct { // CreateMintTx returns the newly created unsigned transaction func (service *Service) CreateMintTx(r *http.Request, args *CreateMintTxArgs, reply *CreateMintTxReply) error { - service.vm.ctx.Log.Debug("AVM: CreateMintTx called") + service.vm.ctx.log.Info("AVM: CreateMintTx called") if args.Amount == 0 { return errInvalidMintAmount @@ -990,7 +990,7 @@ type SignMintTxReply struct { // SignMintTx returns the newly signed transaction func (service *Service) SignMintTx(r *http.Request, args *SignMintTxArgs, reply *SignMintTxReply) error { - service.vm.ctx.Log.Debug("AVM: SignMintTx called") + service.vm.ctx.log.Info("AVM: SignMintTx called") minter, err := service.vm.Parse(args.Minter) if err != nil { @@ -1116,7 +1116,7 @@ type ImportAVAReply struct { // The AVA must have already been exported from the P-Chain. // Returns the ID of the newly created atomic transaction func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, reply *ImportAVAReply) error { - service.vm.ctx.Log.Debug("AVM: ImportAVA called with username: %s", args.Username) + service.vm.ctx.log.Info("AVM: ImportAVA called with username: %s", args.Username) toBytes, err := service.vm.Parse(args.To) if err != nil { @@ -1268,7 +1268,7 @@ type ExportAVAReply struct { // After this tx is accepted, the AVA must be imported to the P-chain with an importTx. // Returns the ID of the newly created atomic transaction func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, reply *ExportAVAReply) error { - service.vm.ctx.Log.Debug("AVM: ExportAVA called with username: %s", args.Username) + service.vm.ctx.log.Info("AVM: ExportAVA called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 52010d1..809ef8a 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -234,7 +234,7 @@ type GetCurrentValidatorsReply struct { // GetCurrentValidators returns the list of current validators func (service *Service) GetCurrentValidators(_ *http.Request, args *GetCurrentValidatorsArgs, reply *GetCurrentValidatorsReply) error { - service.vm.Ctx.Log.Debug("Platform: GetCurrentValidators called") + service.vm.Ctx.log.Info("Platform: GetCurrentValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -298,7 +298,7 @@ type GetPendingValidatorsReply struct { // GetPendingValidators returns the list of current validators func (service *Service) GetPendingValidators(_ *http.Request, args *GetPendingValidatorsArgs, reply *GetPendingValidatorsReply) error { - service.vm.Ctx.Log.Debug("Platform: GetPendingValidators called") + service.vm.Ctx.log.Info("Platform: GetPendingValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -360,7 +360,7 @@ type SampleValidatorsReply struct { // SampleValidators returns a sampling of the list of current validators func (service *Service) SampleValidators(_ *http.Request, args *SampleValidatorsArgs, reply *SampleValidatorsReply) error { - service.vm.Ctx.Log.Debug("Platform: SampleValidators called with {Size = %d}", args.Size) + service.vm.Ctx.log.Info("Platform: SampleValidators called with {Size = %d}", args.Size) if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -437,7 +437,7 @@ type ListAccountsReply struct { // ListAccounts lists all of the accounts controlled by [args.Username] func (service *Service) ListAccounts(_ *http.Request, args *ListAccountsArgs, reply *ListAccountsReply) error { - service.vm.Ctx.Log.Debug("Platform: ListAccounts called for user '%s'", args.Username) + service.vm.Ctx.log.Info("Platform: ListAccounts called for user '%s'", args.Username) // db holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -499,7 +499,7 @@ type CreateAccountReply struct { // The account's ID is [privKey].PublicKey().Address(), where [privKey] is a // private key controlled by the user. func (service *Service) CreateAccount(_ *http.Request, args *CreateAccountArgs, reply *CreateAccountReply) error { - service.vm.Ctx.Log.Debug("Platform: CreateAccount called for user '%s'", args.Username) + service.vm.Ctx.log.Info("Platform: CreateAccount called for user '%s'", args.Username) // userDB holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -569,7 +569,7 @@ type AddDefaultSubnetValidatorArgs struct { // AddDefaultSubnetValidator returns an unsigned transaction to add a validator to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetValidator(_ *http.Request, args *AddDefaultSubnetValidatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: AddDefaultSubnetValidator called") + service.vm.Ctx.log.Info("Platform: AddDefaultSubnetValidator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -626,7 +626,7 @@ type AddDefaultSubnetDelegatorArgs struct { // to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetDelegator(_ *http.Request, args *AddDefaultSubnetDelegatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: AddDefaultSubnetDelegator called") + service.vm.Ctx.log.Info("Platform: AddDefaultSubnetDelegator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -741,7 +741,7 @@ type CreateSubnetArgs struct { // CreateSubnet returns an unsigned transaction to create a new subnet. // The unsigned transaction must be signed with the key of [args.Payer] func (service *Service) CreateSubnet(_ *http.Request, args *CreateSubnetArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: CreateSubnet called") + service.vm.Ctx.log.Info("Platform: CreateSubnet called") switch { case args.PayerNonce == 0: @@ -796,7 +796,7 @@ type ExportAVAArgs struct { // The unsigned transaction must be signed with the key of the account exporting the AVA // and paying the transaction fee func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: ExportAVA called") + service.vm.Ctx.log.Info("Platform: ExportAVA called") switch { case args.PayerNonce == 0: @@ -858,7 +858,7 @@ type SignResponse struct { // Sign [args.bytes] func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignResponse) error { - service.vm.Ctx.Log.Debug("Platform: Sign called") + service.vm.Ctx.log.Info("Platform: Sign called") if args.Signer == "" { return errNilSigner @@ -915,7 +915,7 @@ func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignRespons // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Debug("Platform: signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.log.Info("Platform: signAddDefaultSubnetValidatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetValidatorTx) @@ -938,7 +938,7 @@ func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetVali // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDelegatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetDelegatorTx, error) { - service.vm.Ctx.Log.Debug("Platform: signAddDefaultSubnetDelegatorTx called") + service.vm.Ctx.log.Info("Platform: signAddDefaultSubnetDelegatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx) @@ -961,7 +961,7 @@ func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDele // Sign [xt] with [key] func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.PrivateKeySECP256K1R) (*CreateSubnetTx, error) { - service.vm.Ctx.Log.Debug("Platform: signCreateSubnetTx called") + service.vm.Ctx.log.Info("Platform: signCreateSubnetTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedCreateSubnetTx) @@ -984,7 +984,7 @@ func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.Priva // Sign [tx] with [key] func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256K1R) (*ExportTx, error) { - service.vm.Ctx.Log.Debug("Platform: signExportTx called") + service.vm.Ctx.log.Info("Platform: signExportTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedExportTx) @@ -1012,7 +1012,7 @@ func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256 // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signAddNonDefaultSubnetValidatorTx(tx *addNonDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addNonDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Debug("Platform: signAddNonDefaultSubnetValidatorTx called") + service.vm.Ctx.log.Info("Platform: signAddNonDefaultSubnetValidatorTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedAddNonDefaultSubnetValidatorTx) @@ -1075,7 +1075,7 @@ type ImportAVAArgs struct { // The AVA must have already been exported from the X-Chain. // The unsigned transaction must be signed with the key of the tx fee payer. func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response *SignResponse) error { - service.vm.Ctx.Log.Debug("Platform: ImportAVA called") + service.vm.Ctx.log.Info("Platform: ImportAVA called") switch { case args.To == "": @@ -1203,7 +1203,7 @@ func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signCreateChainTx(tx *CreateChainTx, key *crypto.PrivateKeySECP256K1R) (*CreateChainTx, error) { - service.vm.Ctx.Log.Debug("Platform: signCreateChainTx called") + service.vm.Ctx.log.Info("Platform: signCreateChainTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedCreateChainTx) @@ -1263,7 +1263,7 @@ type IssueTxResponse struct { // IssueTx issues the transaction [args.Tx] to the network func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *IssueTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: IssueTx called") + service.vm.Ctx.log.Info("Platform: IssueTx called") genTx := genericTx{} if err := Codec.Unmarshal(args.Tx.Bytes, &genTx); err != nil { @@ -1327,7 +1327,7 @@ type CreateBlockchainArgs struct { // CreateBlockchain returns an unsigned transaction to create a new blockchain // Must be signed with the Subnet's control keys and with a key that pays the transaction fee before issuance func (service *Service) CreateBlockchain(_ *http.Request, args *CreateBlockchainArgs, response *CreateTxResponse) error { - service.vm.Ctx.Log.Debug("Platform: CreateBlockchain called") + service.vm.Ctx.log.Info("Platform: CreateBlockchain called") switch { case args.PayerNonce == 0: @@ -1410,7 +1410,7 @@ type GetBlockchainStatusReply struct { // GetBlockchainStatus gets the status of a blockchain with the ID [args.BlockchainID]. func (service *Service) GetBlockchainStatus(_ *http.Request, args *GetBlockchainStatusArgs, reply *GetBlockchainStatusReply) error { - service.vm.Ctx.Log.Debug("Platform: GetBlockchainStatus called") + service.vm.Ctx.log.Info("Platform: GetBlockchainStatus called") switch { case args.BlockchainID == "": @@ -1490,7 +1490,7 @@ type ValidatedByResponse struct { // ValidatedBy returns the ID of the Subnet that validates [args.BlockchainID] func (service *Service) ValidatedBy(_ *http.Request, args *ValidatedByArgs, response *ValidatedByResponse) error { - service.vm.Ctx.Log.Debug("Platform: ValidatedBy called") + service.vm.Ctx.log.Info("Platform: ValidatedBy called") switch { case args.BlockchainID == "": @@ -1522,7 +1522,7 @@ type ValidatesResponse struct { // Validates returns the IDs of the blockchains validated by [args.SubnetID] func (service *Service) Validates(_ *http.Request, args *ValidatesArgs, response *ValidatesResponse) error { - service.vm.Ctx.Log.Debug("Platform: Validates called") + service.vm.Ctx.log.Info("Platform: Validates called") switch { case args.SubnetID == "": @@ -1576,7 +1576,7 @@ type GetBlockchainsResponse struct { // GetBlockchains returns all of the blockchains that exist func (service *Service) GetBlockchains(_ *http.Request, args *struct{}, response *GetBlockchainsResponse) error { - service.vm.Ctx.Log.Debug("Platform: GetBlockchains called") + service.vm.Ctx.log.Info("Platform: GetBlockchains called") chains, err := service.vm.getChains(service.vm.DB) if err != nil { From cdac10c23b64e6c331f7d8d2bd98ae2ae2578481 Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Mon, 15 Jun 2020 09:45:21 -0700 Subject: [PATCH 33/42] Fix typos. --- vms/avm/service.go | 34 ++++++++++++++--------------- vms/platformvm/service.go | 46 +++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/vms/avm/service.go b/vms/avm/service.go index 02f0216..a9fab92 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -56,7 +56,7 @@ type IssueTxReply struct { // IssueTx attempts to issue a transaction into consensus func (service *Service) IssueTx(r *http.Request, args *IssueTxArgs, reply *IssueTxReply) error { - service.vm.ctx.log.Info("AVM: IssueTx called with %s", args.Tx) + service.vm.ctx.Log.Info("AVM: IssueTx called with %s", args.Tx) txID, err := service.vm.IssueTx(args.Tx.Bytes, nil) if err != nil { @@ -79,7 +79,7 @@ type GetTxStatusReply struct { // GetTxStatus returns the status of the specified transaction func (service *Service) GetTxStatus(r *http.Request, args *GetTxStatusArgs, reply *GetTxStatusReply) error { - service.vm.ctx.log.Info("AVM: GetTxStatus called with %s", args.TxID) + service.vm.ctx.Log.Info("AVM: GetTxStatus called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -106,7 +106,7 @@ type GetTxReply struct { // GetTx returns the specified transaction func (service *Service) GetTx(r *http.Request, args *GetTxArgs, reply *GetTxReply) error { - service.vm.ctx.log.Info("AVM: GetTx called with %s", args.TxID) + service.vm.ctx.Log.Info("AVM: GetTx called with %s", args.TxID) if args.TxID.IsZero() { return errNilTxID @@ -136,7 +136,7 @@ type GetUTXOsReply struct { // GetUTXOs creates an empty account with the name passed in func (service *Service) GetUTXOs(r *http.Request, args *GetUTXOsArgs, reply *GetUTXOsReply) error { - service.vm.ctx.log.Info("AVM: GetUTXOs called with %s", args.Addresses) + service.vm.ctx.Log.Info("AVM: GetUTXOs called with %s", args.Addresses) addrSet := ids.Set{} for _, addr := range args.Addresses { @@ -178,7 +178,7 @@ type GetAssetDescriptionReply struct { // GetAssetDescription creates an empty account with the name passed in func (service *Service) GetAssetDescription(_ *http.Request, args *GetAssetDescriptionArgs, reply *GetAssetDescriptionReply) error { - service.vm.ctx.log.Info("AVM: GetAssetDescription called with %s", args.AssetID) + service.vm.ctx.Log.Info("AVM: GetAssetDescription called with %s", args.AssetID) assetID, err := service.vm.Lookup(args.AssetID) if err != nil { @@ -222,7 +222,7 @@ type GetBalanceReply struct { // GetBalance returns the amount of an asset that an address at least partially owns func (service *Service) GetBalance(r *http.Request, args *GetBalanceArgs, reply *GetBalanceReply) error { - service.vm.ctx.log.Info("AVM: GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) + service.vm.ctx.Log.Info("AVM: GetBalance called with address: %s assetID: %s", args.Address, args.AssetID) address, err := service.vm.Parse(args.Address) if err != nil { @@ -287,7 +287,7 @@ type GetAllBalancesReply struct { // Note that balances include assets that the address only _partially_ owns // (ie is one of several addresses specified in a multi-sig) func (service *Service) GetAllBalances(r *http.Request, args *GetAllBalancesArgs, reply *GetAllBalancesReply) error { - service.vm.ctx.log.Info("AVM: GetAllBalances called with address: %s", args.Address) + service.vm.ctx.Log.Info("AVM: GetAllBalances called with address: %s", args.Address) address, err := service.vm.Parse(args.Address) if err != nil { @@ -360,7 +360,7 @@ type CreateFixedCapAssetReply struct { // CreateFixedCapAsset returns ID of the newly created asset func (service *Service) CreateFixedCapAsset(r *http.Request, args *CreateFixedCapAssetArgs, reply *CreateFixedCapAssetReply) error { - service.vm.ctx.log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", + service.vm.ctx.Log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of holders: %d", args.Name, args.Symbol, len(args.InitialHolders), @@ -445,7 +445,7 @@ type CreateVariableCapAssetReply struct { // CreateVariableCapAsset returns ID of the newly created asset func (service *Service) CreateVariableCapAsset(r *http.Request, args *CreateVariableCapAssetArgs, reply *CreateVariableCapAssetReply) error { - service.vm.ctx.log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", + service.vm.ctx.Log.Info("AVM: CreateFixedCapAsset called with name: %s symbol: %s number of minters: %d", args.Name, args.Symbol, len(args.MinterSets), @@ -523,7 +523,7 @@ type CreateAddressReply struct { // CreateAddress creates an address for the user [args.Username] func (service *Service) CreateAddress(r *http.Request, args *CreateAddressArgs, reply *CreateAddressReply) error { - service.vm.ctx.log.Info("AVM: CreateAddress called for user '%s'", args.Username) + service.vm.ctx.Log.Info("AVM: CreateAddress called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -603,7 +603,7 @@ type ExportKeyReply struct { // ExportKey returns a private key from the provided user func (service *Service) ExportKey(r *http.Request, args *ExportKeyArgs, reply *ExportKeyReply) error { - service.vm.ctx.log.Info("AVM: ExportKey called for user '%s'", args.Username) + service.vm.ctx.Log.Info("AVM: ExportKey called for user '%s'", args.Username) address, err := service.vm.Parse(args.Address) if err != nil { @@ -645,7 +645,7 @@ type ImportKeyReply struct { // ImportKey adds a private key to the provided user func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *ImportKeyReply) error { - service.vm.ctx.log.Info("AVM: ImportKey called for user '%s'", args.Username) + service.vm.ctx.Log.Info("AVM: ImportKey called for user '%s'", args.Username) db, err := service.vm.ctx.Keystore.GetDatabase(args.Username, args.Password) if err != nil { @@ -692,7 +692,7 @@ type SendReply struct { // Send returns the ID of the newly created transaction func (service *Service) Send(r *http.Request, args *SendArgs, reply *SendReply) error { - service.vm.ctx.log.Info("AVM: Send called with username: %s", args.Username) + service.vm.ctx.Log.Info("AVM: Send called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount @@ -873,7 +873,7 @@ type CreateMintTxReply struct { // CreateMintTx returns the newly created unsigned transaction func (service *Service) CreateMintTx(r *http.Request, args *CreateMintTxArgs, reply *CreateMintTxReply) error { - service.vm.ctx.log.Info("AVM: CreateMintTx called") + service.vm.ctx.Log.Info("AVM: CreateMintTx called") if args.Amount == 0 { return errInvalidMintAmount @@ -990,7 +990,7 @@ type SignMintTxReply struct { // SignMintTx returns the newly signed transaction func (service *Service) SignMintTx(r *http.Request, args *SignMintTxArgs, reply *SignMintTxReply) error { - service.vm.ctx.log.Info("AVM: SignMintTx called") + service.vm.ctx.Log.Info("AVM: SignMintTx called") minter, err := service.vm.Parse(args.Minter) if err != nil { @@ -1116,7 +1116,7 @@ type ImportAVAReply struct { // The AVA must have already been exported from the P-Chain. // Returns the ID of the newly created atomic transaction func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, reply *ImportAVAReply) error { - service.vm.ctx.log.Info("AVM: ImportAVA called with username: %s", args.Username) + service.vm.ctx.Log.Info("AVM: ImportAVA called with username: %s", args.Username) toBytes, err := service.vm.Parse(args.To) if err != nil { @@ -1268,7 +1268,7 @@ type ExportAVAReply struct { // After this tx is accepted, the AVA must be imported to the P-chain with an importTx. // Returns the ID of the newly created atomic transaction func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, reply *ExportAVAReply) error { - service.vm.ctx.log.Info("AVM: ExportAVA called with username: %s", args.Username) + service.vm.ctx.Log.Info("AVM: ExportAVA called with username: %s", args.Username) if args.Amount == 0 { return errInvalidAmount diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 809ef8a..69cfb67 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -234,7 +234,7 @@ type GetCurrentValidatorsReply struct { // GetCurrentValidators returns the list of current validators func (service *Service) GetCurrentValidators(_ *http.Request, args *GetCurrentValidatorsArgs, reply *GetCurrentValidatorsReply) error { - service.vm.Ctx.log.Info("Platform: GetCurrentValidators called") + service.vm.Ctx.Log.Info("Platform: GetCurrentValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -298,7 +298,7 @@ type GetPendingValidatorsReply struct { // GetPendingValidators returns the list of current validators func (service *Service) GetPendingValidators(_ *http.Request, args *GetPendingValidatorsArgs, reply *GetPendingValidatorsReply) error { - service.vm.Ctx.log.Info("Platform: GetPendingValidators called") + service.vm.Ctx.Log.Info("Platform: GetPendingValidators called") if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -360,7 +360,7 @@ type SampleValidatorsReply struct { // SampleValidators returns a sampling of the list of current validators func (service *Service) SampleValidators(_ *http.Request, args *SampleValidatorsArgs, reply *SampleValidatorsReply) error { - service.vm.Ctx.log.Info("Platform: SampleValidators called with {Size = %d}", args.Size) + service.vm.Ctx.Log.Info("Platform: SampleValidators called with {Size = %d}", args.Size) if args.SubnetID.IsZero() { args.SubnetID = DefaultSubnetID @@ -437,7 +437,7 @@ type ListAccountsReply struct { // ListAccounts lists all of the accounts controlled by [args.Username] func (service *Service) ListAccounts(_ *http.Request, args *ListAccountsArgs, reply *ListAccountsReply) error { - service.vm.Ctx.log.Info("Platform: ListAccounts called for user '%s'", args.Username) + service.vm.Ctx.Log.Info("Platform: ListAccounts called for user '%s'", args.Username) // db holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -499,7 +499,7 @@ type CreateAccountReply struct { // The account's ID is [privKey].PublicKey().Address(), where [privKey] is a // private key controlled by the user. func (service *Service) CreateAccount(_ *http.Request, args *CreateAccountArgs, reply *CreateAccountReply) error { - service.vm.Ctx.log.Info("Platform: CreateAccount called for user '%s'", args.Username) + service.vm.Ctx.Log.Info("Platform: CreateAccount called for user '%s'", args.Username) // userDB holds the user's info that pertains to the Platform Chain userDB, err := service.vm.Ctx.Keystore.GetDatabase(args.Username, args.Password) @@ -569,7 +569,7 @@ type AddDefaultSubnetValidatorArgs struct { // AddDefaultSubnetValidator returns an unsigned transaction to add a validator to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetValidator(_ *http.Request, args *AddDefaultSubnetValidatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.log.Info("Platform: AddDefaultSubnetValidator called") + service.vm.Ctx.Log.Info("Platform: AddDefaultSubnetValidator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -626,7 +626,7 @@ type AddDefaultSubnetDelegatorArgs struct { // to the default subnet // The returned unsigned transaction should be signed using Sign() func (service *Service) AddDefaultSubnetDelegator(_ *http.Request, args *AddDefaultSubnetDelegatorArgs, reply *CreateTxResponse) error { - service.vm.Ctx.log.Info("Platform: AddDefaultSubnetDelegator called") + service.vm.Ctx.Log.Info("Platform: AddDefaultSubnetDelegator called") switch { case args.ID.IsZero(): // If ID unspecified, use this node's ID as validator ID @@ -741,7 +741,7 @@ type CreateSubnetArgs struct { // CreateSubnet returns an unsigned transaction to create a new subnet. // The unsigned transaction must be signed with the key of [args.Payer] func (service *Service) CreateSubnet(_ *http.Request, args *CreateSubnetArgs, response *CreateTxResponse) error { - service.vm.Ctx.log.Info("Platform: CreateSubnet called") + service.vm.Ctx.Log.Info("Platform: CreateSubnet called") switch { case args.PayerNonce == 0: @@ -796,7 +796,7 @@ type ExportAVAArgs struct { // The unsigned transaction must be signed with the key of the account exporting the AVA // and paying the transaction fee func (service *Service) ExportAVA(_ *http.Request, args *ExportAVAArgs, response *CreateTxResponse) error { - service.vm.Ctx.log.Info("Platform: ExportAVA called") + service.vm.Ctx.Log.Info("Platform: ExportAVA called") switch { case args.PayerNonce == 0: @@ -858,7 +858,7 @@ type SignResponse struct { // Sign [args.bytes] func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignResponse) error { - service.vm.Ctx.log.Info("Platform: Sign called") + service.vm.Ctx.Log.Info("Platform: Sign called") if args.Signer == "" { return errNilSigner @@ -915,7 +915,7 @@ func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignRespons // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetValidatorTx, error) { - service.vm.Ctx.log.Info("Platform: signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Info("Platform: signAddDefaultSubnetValidatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetValidatorTx) @@ -938,7 +938,7 @@ func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetVali // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDelegatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetDelegatorTx, error) { - service.vm.Ctx.log.Info("Platform: signAddDefaultSubnetDelegatorTx called") + service.vm.Ctx.Log.Info("Platform: signAddDefaultSubnetDelegatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx) @@ -961,7 +961,7 @@ func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDele // Sign [xt] with [key] func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.PrivateKeySECP256K1R) (*CreateSubnetTx, error) { - service.vm.Ctx.log.Info("Platform: signCreateSubnetTx called") + service.vm.Ctx.Log.Info("Platform: signCreateSubnetTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedCreateSubnetTx) @@ -984,7 +984,7 @@ func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.Priva // Sign [tx] with [key] func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256K1R) (*ExportTx, error) { - service.vm.Ctx.log.Info("Platform: signExportTx called") + service.vm.Ctx.Log.Info("Platform: signExportTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedExportTx) @@ -1012,7 +1012,7 @@ func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256 // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signAddNonDefaultSubnetValidatorTx(tx *addNonDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addNonDefaultSubnetValidatorTx, error) { - service.vm.Ctx.log.Info("Platform: signAddNonDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Info("Platform: signAddNonDefaultSubnetValidatorTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedAddNonDefaultSubnetValidatorTx) @@ -1075,7 +1075,7 @@ type ImportAVAArgs struct { // The AVA must have already been exported from the X-Chain. // The unsigned transaction must be signed with the key of the tx fee payer. func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response *SignResponse) error { - service.vm.Ctx.log.Info("Platform: ImportAVA called") + service.vm.Ctx.Log.Info("Platform: ImportAVA called") switch { case args.To == "": @@ -1203,7 +1203,7 @@ func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signCreateChainTx(tx *CreateChainTx, key *crypto.PrivateKeySECP256K1R) (*CreateChainTx, error) { - service.vm.Ctx.log.Info("Platform: signCreateChainTx called") + service.vm.Ctx.Log.Info("Platform: signCreateChainTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedCreateChainTx) @@ -1263,7 +1263,7 @@ type IssueTxResponse struct { // IssueTx issues the transaction [args.Tx] to the network func (service *Service) IssueTx(_ *http.Request, args *IssueTxArgs, response *IssueTxResponse) error { - service.vm.Ctx.log.Info("Platform: IssueTx called") + service.vm.Ctx.Log.Info("Platform: IssueTx called") genTx := genericTx{} if err := Codec.Unmarshal(args.Tx.Bytes, &genTx); err != nil { @@ -1327,7 +1327,7 @@ type CreateBlockchainArgs struct { // CreateBlockchain returns an unsigned transaction to create a new blockchain // Must be signed with the Subnet's control keys and with a key that pays the transaction fee before issuance func (service *Service) CreateBlockchain(_ *http.Request, args *CreateBlockchainArgs, response *CreateTxResponse) error { - service.vm.Ctx.log.Info("Platform: CreateBlockchain called") + service.vm.Ctx.Log.Info("Platform: CreateBlockchain called") switch { case args.PayerNonce == 0: @@ -1410,7 +1410,7 @@ type GetBlockchainStatusReply struct { // GetBlockchainStatus gets the status of a blockchain with the ID [args.BlockchainID]. func (service *Service) GetBlockchainStatus(_ *http.Request, args *GetBlockchainStatusArgs, reply *GetBlockchainStatusReply) error { - service.vm.Ctx.log.Info("Platform: GetBlockchainStatus called") + service.vm.Ctx.Log.Info("Platform: GetBlockchainStatus called") switch { case args.BlockchainID == "": @@ -1490,7 +1490,7 @@ type ValidatedByResponse struct { // ValidatedBy returns the ID of the Subnet that validates [args.BlockchainID] func (service *Service) ValidatedBy(_ *http.Request, args *ValidatedByArgs, response *ValidatedByResponse) error { - service.vm.Ctx.log.Info("Platform: ValidatedBy called") + service.vm.Ctx.Log.Info("Platform: ValidatedBy called") switch { case args.BlockchainID == "": @@ -1522,7 +1522,7 @@ type ValidatesResponse struct { // Validates returns the IDs of the blockchains validated by [args.SubnetID] func (service *Service) Validates(_ *http.Request, args *ValidatesArgs, response *ValidatesResponse) error { - service.vm.Ctx.log.Info("Platform: Validates called") + service.vm.Ctx.Log.Info("Platform: Validates called") switch { case args.SubnetID == "": @@ -1576,7 +1576,7 @@ type GetBlockchainsResponse struct { // GetBlockchains returns all of the blockchains that exist func (service *Service) GetBlockchains(_ *http.Request, args *struct{}, response *GetBlockchainsResponse) error { - service.vm.Ctx.log.Info("Platform: GetBlockchains called") + service.vm.Ctx.Log.Info("Platform: GetBlockchains called") chains, err := service.vm.getChains(service.vm.DB) if err != nil { From acbb9a7e0cbde173b59a12ebf72b4da4464669bd Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 13:12:55 -0400 Subject: [PATCH 34/42] remove expansionBoost from packer (Go's append does similar already). change initialSliceCap 1024 --> 256. Streamline packer.Expand, as this method is called very often --- utils/wrappers/packing.go | 32 ++++++++------------------------ vms/components/codec/codec.go | 2 +- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/utils/wrappers/packing.go b/utils/wrappers/packing.go index 22c7464..c048f9c 100644 --- a/utils/wrappers/packing.go +++ b/utils/wrappers/packing.go @@ -16,11 +16,6 @@ const ( // MaxStringLen ... MaxStringLen = math.MaxUint16 - // When the byte array is expanded, this many extra bytes - // are added to capacity of the array. - // Higher value --> need to expand byte array less --> less memory allocations - expansionBoost = 256 - // ByteLen is the number of bytes per byte... ByteLen = 1 // ShortLen is the number of bytes per short @@ -71,30 +66,19 @@ func (p *Packer) CheckSpace(bytes int) { // In order to understand this code, its important to understand the difference // between a slice's length and its capacity. func (p *Packer) Expand(bytes int) { - p.CheckSpace(0) - if p.Errored() { + neededSize := bytes + p.Offset // Need byte slice's length to be at least [neededSize] + switch { + case neededSize <= len(p.Bytes): // Byte slice has sufficient length already return - } - - neededSize := bytes + p.Offset // Need byte slice's length to be at least [neededSize] - if neededSize <= len(p.Bytes) { // Byte slice has sufficient length already + case neededSize > p.MaxSize: // Lengthening the byte slice would cause it to grow too large + p.Err = errBadLength return - } else if neededSize > p.MaxSize { // Lengthening the byte slice would cause it to grow too large - p.Add(errBadLength) - return - } else if neededSize <= cap(p.Bytes) { // Byte slice has sufficient capacity to lengthen it without mem alloc + case neededSize <= cap(p.Bytes): // Byte slice has sufficient capacity to lengthen it without mem alloc p.Bytes = p.Bytes[:neededSize] return + default: // Add capacity/length to byte slice + p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes))...) } - - // See if we can expand the byte slice an extra [expansionBoost] bytes in order to - // prevent need for future expansions (and therefore memory allocations) - capToAdd := neededSize - cap(p.Bytes) + expansionBoost - if capToAdd > p.MaxSize { - capToAdd = neededSize - cap(p.Bytes) - } - // increase slice's length and capacity - p.Bytes = append(p.Bytes[:cap(p.Bytes)], make([]byte, neededSize-cap(p.Bytes), capToAdd)...) } // PackByte append a byte to the byte array diff --git a/vms/components/codec/codec.go b/vms/components/codec/codec.go index 53852a9..6521993 100644 --- a/vms/components/codec/codec.go +++ b/vms/components/codec/codec.go @@ -19,7 +19,7 @@ const ( // initial capacity of byte slice that values are marshaled into. // Larger value --> need less memory allocations but possibly have allocated but unused memory // Smaller value --> need more memory allocations but more efficient use of allocated memory - initialSliceCap = 1024 + initialSliceCap = 256 ) var ( From fa4be45d8aa8c8097c830eee3169af50202063dd Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Mon, 15 Jun 2020 10:15:43 -0700 Subject: [PATCH 35/42] Update go.sum --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index 774be35..d79e9a8 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/AppsFlyer/go-sundheit v0.2.0 h1:FArqX+HbqZ6U32RC3giEAWRUpkggqxHj91KIvxNgwjU= github.com/AppsFlyer/go-sundheit v0.2.0/go.mod h1:rCRkVTMQo7/krF7xQ9X0XEF1an68viFR6/Gy02q+4ds= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= From 18c0b0a65b62880ba1268d0a33b095b44517e84d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 15 Jun 2020 13:20:30 -0400 Subject: [PATCH 36/42] move codec to utils --- api/keystore/service.go | 2 +- chains/atomic/memory.go | 2 +- database/encdb/db.go | 2 +- genesis/genesis.go | 2 +- {vms/components => utils}/codec/codec.go | 0 {vms/components => utils}/codec/codec_benchmark_test.go | 0 {vms/components => utils}/codec/codec_test.go | 0 vms/avm/base_tx.go | 2 +- vms/avm/create_asset_tx.go | 2 +- vms/avm/create_asset_tx_test.go | 2 +- vms/avm/export_tx.go | 2 +- vms/avm/export_tx_test.go | 2 +- vms/avm/import_tx.go | 2 +- vms/avm/initial_state.go | 2 +- vms/avm/initial_state_test.go | 2 +- vms/avm/operation.go | 2 +- vms/avm/operation_test.go | 2 +- vms/avm/operation_tx.go | 2 +- vms/avm/static_service.go | 2 +- vms/avm/tx.go | 2 +- vms/avm/tx_test.go | 2 +- vms/avm/vm.go | 2 +- vms/components/ava/asset_test.go | 2 +- vms/components/ava/prefixed_state.go | 2 +- vms/components/ava/prefixed_state_test.go | 2 +- vms/components/ava/state.go | 2 +- vms/components/ava/transferables.go | 2 +- vms/components/ava/transferables_test.go | 2 +- vms/components/ava/utxo_id_test.go | 2 +- vms/components/ava/utxo_test.go | 2 +- vms/nftfx/fx_test.go | 2 +- vms/platformvm/vm.go | 2 +- vms/propertyfx/fx_test.go | 2 +- vms/secp256k1fx/credential_test.go | 2 +- vms/secp256k1fx/fx_test.go | 2 +- vms/secp256k1fx/transer_input_test.go | 2 +- vms/secp256k1fx/transfer_output_test.go | 2 +- vms/secp256k1fx/vm.go | 2 +- vms/timestampvm/vm.go | 2 +- xputtest/avmwallet/wallet.go | 2 +- 40 files changed, 37 insertions(+), 37 deletions(-) rename {vms/components => utils}/codec/codec.go (100%) rename {vms/components => utils}/codec/codec_benchmark_test.go (100%) rename {vms/components => utils}/codec/codec_test.go (100%) diff --git a/api/keystore/service.go b/api/keystore/service.go index 16aca06..25e9a02 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -19,7 +19,7 @@ import ( "github.com/ava-labs/gecko/snow/engine/common" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/logging" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" jsoncodec "github.com/ava-labs/gecko/utils/json" zxcvbn "github.com/nbutton23/zxcvbn-go" diff --git a/chains/atomic/memory.go b/chains/atomic/memory.go index 448e6c9..9774711 100644 --- a/chains/atomic/memory.go +++ b/chains/atomic/memory.go @@ -12,7 +12,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) type rcLock struct { diff --git a/database/encdb/db.go b/database/encdb/db.go index eb06549..4814805 100644 --- a/database/encdb/db.go +++ b/database/encdb/db.go @@ -14,7 +14,7 @@ import ( "github.com/ava-labs/gecko/database/nodb" "github.com/ava-labs/gecko/utils" "github.com/ava-labs/gecko/utils/hashing" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) // Database encrypts all values that are provided diff --git a/genesis/genesis.go b/genesis/genesis.go index 4cad047..c4245b9 100644 --- a/genesis/genesis.go +++ b/genesis/genesis.go @@ -14,7 +14,7 @@ import ( "github.com/ava-labs/gecko/utils/units" "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/avm" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/nftfx" "github.com/ava-labs/gecko/vms/platformvm" "github.com/ava-labs/gecko/vms/propertyfx" diff --git a/vms/components/codec/codec.go b/utils/codec/codec.go similarity index 100% rename from vms/components/codec/codec.go rename to utils/codec/codec.go diff --git a/vms/components/codec/codec_benchmark_test.go b/utils/codec/codec_benchmark_test.go similarity index 100% rename from vms/components/codec/codec_benchmark_test.go rename to utils/codec/codec_benchmark_test.go diff --git a/vms/components/codec/codec_test.go b/utils/codec/codec_test.go similarity index 100% rename from vms/components/codec/codec_test.go rename to utils/codec/codec_test.go diff --git a/vms/avm/base_tx.go b/vms/avm/base_tx.go index 33cba51..0ab3fa4 100644 --- a/vms/avm/base_tx.go +++ b/vms/avm/base_tx.go @@ -10,7 +10,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/create_asset_tx.go b/vms/avm/create_asset_tx.go index 9f95a15..77aae2f 100644 --- a/vms/avm/create_asset_tx.go +++ b/vms/avm/create_asset_tx.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) const ( diff --git a/vms/avm/create_asset_tx_test.go b/vms/avm/create_asset_tx_test.go index a26a815..324f403 100644 --- a/vms/avm/create_asset_tx_test.go +++ b/vms/avm/create_asset_tx_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/avm/export_tx.go b/vms/avm/export_tx.go index d5222f4..d788360 100644 --- a/vms/avm/export_tx.go +++ b/vms/avm/export_tx.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/database/versiondb" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/export_tx_test.go b/vms/avm/export_tx_test.go index 4e9d064..fdef399 100644 --- a/vms/avm/export_tx_test.go +++ b/vms/avm/export_tx_test.go @@ -16,7 +16,7 @@ import ( "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/avm/import_tx.go b/vms/avm/import_tx.go index 09dec6e..1729221 100644 --- a/vms/avm/import_tx.go +++ b/vms/avm/import_tx.go @@ -12,7 +12,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/initial_state.go b/vms/avm/initial_state.go index c3d4b16..73ad6e4 100644 --- a/vms/avm/initial_state.go +++ b/vms/avm/initial_state.go @@ -8,7 +8,7 @@ import ( "errors" "sort" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/initial_state_test.go b/vms/avm/initial_state_test.go index 67c4b15..b61876c 100644 --- a/vms/avm/initial_state_test.go +++ b/vms/avm/initial_state_test.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/avm/operation.go b/vms/avm/operation.go index 3b5fc9a..ef9317b 100644 --- a/vms/avm/operation.go +++ b/vms/avm/operation.go @@ -10,7 +10,7 @@ import ( "github.com/ava-labs/gecko/utils" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/operation_test.go b/vms/avm/operation_test.go index 8948388..8b85901 100644 --- a/vms/avm/operation_test.go +++ b/vms/avm/operation_test.go @@ -8,7 +8,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/operation_tx.go b/vms/avm/operation_tx.go index 9384f8d..ec419c7 100644 --- a/vms/avm/operation_tx.go +++ b/vms/avm/operation_tx.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/static_service.go b/vms/avm/static_service.go index 3fd58f3..48b58a9 100644 --- a/vms/avm/static_service.go +++ b/vms/avm/static_service.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/wrappers" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" cjson "github.com/ava-labs/gecko/utils/json" diff --git a/vms/avm/tx.go b/vms/avm/tx.go index c35fd80..f1d0b71 100644 --- a/vms/avm/tx.go +++ b/vms/avm/tx.go @@ -10,7 +10,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/avm/tx_test.go b/vms/avm/tx_test.go index 2f269e9..53e20de 100644 --- a/vms/avm/tx_test.go +++ b/vms/avm/tx_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/units" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/avm/vm.go b/vms/avm/vm.go index b7f7252..4c0820d 100644 --- a/vms/avm/vm.go +++ b/vms/avm/vm.go @@ -25,7 +25,7 @@ import ( "github.com/ava-labs/gecko/utils/timer" "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" cjson "github.com/ava-labs/gecko/utils/json" ) diff --git a/vms/components/ava/asset_test.go b/vms/components/ava/asset_test.go index 40d6ea8..79ae7d5 100644 --- a/vms/components/ava/asset_test.go +++ b/vms/components/ava/asset_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) func TestAssetVerifyNil(t *testing.T) { diff --git a/vms/components/ava/prefixed_state.go b/vms/components/ava/prefixed_state.go index 92b3491..9906381 100644 --- a/vms/components/ava/prefixed_state.go +++ b/vms/components/ava/prefixed_state.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow/choices" "github.com/ava-labs/gecko/utils/hashing" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) // Addressable is the interface a feature extension must provide to be able to diff --git a/vms/components/ava/prefixed_state_test.go b/vms/components/ava/prefixed_state_test.go index 06cb1df..d3019d5 100644 --- a/vms/components/ava/prefixed_state_test.go +++ b/vms/components/ava/prefixed_state_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/database/memdb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/hashing" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/stretchr/testify/assert" ) diff --git a/vms/components/ava/state.go b/vms/components/ava/state.go index fc3b929..df724a4 100644 --- a/vms/components/ava/state.go +++ b/vms/components/ava/state.go @@ -11,7 +11,7 @@ import ( "github.com/ava-labs/gecko/database/prefixdb" "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/snow/choices" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) var ( diff --git a/vms/components/ava/transferables.go b/vms/components/ava/transferables.go index 4aa906d..85c2414 100644 --- a/vms/components/ava/transferables.go +++ b/vms/components/ava/transferables.go @@ -10,7 +10,7 @@ import ( "github.com/ava-labs/gecko/utils" "github.com/ava-labs/gecko/utils/crypto" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/verify" ) diff --git a/vms/components/ava/transferables_test.go b/vms/components/ava/transferables_test.go index 80205a6..08d7b69 100644 --- a/vms/components/ava/transferables_test.go +++ b/vms/components/ava/transferables_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/formatting" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/components/ava/utxo_id_test.go b/vms/components/ava/utxo_id_test.go index 7944961..d1be00f 100644 --- a/vms/components/ava/utxo_id_test.go +++ b/vms/components/ava/utxo_id_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) func TestUTXOIDVerifyNil(t *testing.T) { diff --git a/vms/components/ava/utxo_test.go b/vms/components/ava/utxo_test.go index 07b067a..151e219 100644 --- a/vms/components/ava/utxo_test.go +++ b/vms/components/ava/utxo_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/ids" "github.com/ava-labs/gecko/utils/formatting" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/nftfx/fx_test.go b/vms/nftfx/fx_test.go index d965902..0cfbd87 100644 --- a/vms/nftfx/fx_test.go +++ b/vms/nftfx/fx_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/timer" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index 9f1ce53..1a945cb 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -27,7 +27,7 @@ import ( "github.com/ava-labs/gecko/utils/units" "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/core" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/propertyfx/fx_test.go b/vms/propertyfx/fx_test.go index cfdf5c9..887cf73 100644 --- a/vms/propertyfx/fx_test.go +++ b/vms/propertyfx/fx_test.go @@ -9,7 +9,7 @@ import ( "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/timer" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) diff --git a/vms/secp256k1fx/credential_test.go b/vms/secp256k1fx/credential_test.go index 5157fab..e85ce1b 100644 --- a/vms/secp256k1fx/credential_test.go +++ b/vms/secp256k1fx/credential_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ava-labs/gecko/utils/crypto" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) func TestCredentialVerify(t *testing.T) { diff --git a/vms/secp256k1fx/fx_test.go b/vms/secp256k1fx/fx_test.go index 79e6c89..566b4cb 100644 --- a/vms/secp256k1fx/fx_test.go +++ b/vms/secp256k1fx/fx_test.go @@ -12,7 +12,7 @@ import ( "github.com/ava-labs/gecko/utils/hashing" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/timer" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) var ( diff --git a/vms/secp256k1fx/transer_input_test.go b/vms/secp256k1fx/transer_input_test.go index e954af0..00e894f 100644 --- a/vms/secp256k1fx/transer_input_test.go +++ b/vms/secp256k1fx/transer_input_test.go @@ -7,7 +7,7 @@ import ( "bytes" "testing" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) func TestTransferInputAmount(t *testing.T) { diff --git a/vms/secp256k1fx/transfer_output_test.go b/vms/secp256k1fx/transfer_output_test.go index 7e87875..09bb0ce 100644 --- a/vms/secp256k1fx/transfer_output_test.go +++ b/vms/secp256k1fx/transfer_output_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ava-labs/gecko/ids" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) func TestOutputAmount(t *testing.T) { diff --git a/vms/secp256k1fx/vm.go b/vms/secp256k1fx/vm.go index bb59166..37aa23b 100644 --- a/vms/secp256k1fx/vm.go +++ b/vms/secp256k1fx/vm.go @@ -6,7 +6,7 @@ package secp256k1fx import ( "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/timer" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" ) // VM that this Fx must be run by diff --git a/vms/timestampvm/vm.go b/vms/timestampvm/vm.go index c571d9a..5376e2f 100644 --- a/vms/timestampvm/vm.go +++ b/vms/timestampvm/vm.go @@ -12,7 +12,7 @@ import ( "github.com/ava-labs/gecko/snow" "github.com/ava-labs/gecko/snow/consensus/snowman" "github.com/ava-labs/gecko/snow/engine/common" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/components/core" ) diff --git a/xputtest/avmwallet/wallet.go b/xputtest/avmwallet/wallet.go index ef01eb0..c5d2cd9 100644 --- a/xputtest/avmwallet/wallet.go +++ b/xputtest/avmwallet/wallet.go @@ -19,7 +19,7 @@ import ( "github.com/ava-labs/gecko/utils/wrappers" "github.com/ava-labs/gecko/vms/avm" "github.com/ava-labs/gecko/vms/components/ava" - "github.com/ava-labs/gecko/vms/components/codec" + "github.com/ava-labs/gecko/utils/codec" "github.com/ava-labs/gecko/vms/secp256k1fx" ) From 8783844aca5cfde531120dc83a7d04bccde614f9 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Mon, 15 Jun 2020 14:20:16 -0400 Subject: [PATCH 37/42] Fix nitpick --- vms/avm/service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vms/avm/service.go b/vms/avm/service.go index 37a37a3..039e07f 100644 --- a/vms/avm/service.go +++ b/vms/avm/service.go @@ -668,9 +668,9 @@ func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *I addresses, _ := user.Addresses(db) newAddress := sk.PublicKey().Address() + reply.Address = service.vm.Format(newAddress.Bytes()) for _, address := range addresses { if newAddress.Equals(address) { - reply.Address = service.vm.Format(newAddress.Bytes()) return nil } } @@ -680,7 +680,6 @@ func (service *Service) ImportKey(r *http.Request, args *ImportKeyArgs, reply *I return fmt.Errorf("problem saving addresses: %w", err) } - reply.Address = service.vm.Format(newAddress.Bytes()) return nil } From c3c9cec1ea7b2f1dede07168a32732712d30b0d7 Mon Sep 17 00:00:00 2001 From: StephenButtolph Date: Mon, 15 Jun 2020 15:51:12 -0400 Subject: [PATCH 38/42] updated new messages to match new logging format --- network/network.go | 3 +-- .../router/{subnet_router.go => chain_router.go} | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) rename snow/networking/router/{subnet_router.go => chain_router.go} (97%) diff --git a/network/network.go b/network/network.go index 3fc6893..a280731 100644 --- a/network/network.go +++ b/network/network.go @@ -12,8 +12,6 @@ import ( "sync/atomic" "time" - "github.com/ava-labs/gecko/utils/formatting" - "github.com/prometheus/client_golang/prometheus" "github.com/ava-labs/gecko/api/health" @@ -23,6 +21,7 @@ import ( "github.com/ava-labs/gecko/snow/triggers" "github.com/ava-labs/gecko/snow/validators" "github.com/ava-labs/gecko/utils" + "github.com/ava-labs/gecko/utils/formatting" "github.com/ava-labs/gecko/utils/logging" "github.com/ava-labs/gecko/utils/random" "github.com/ava-labs/gecko/utils/timer" diff --git a/snow/networking/router/subnet_router.go b/snow/networking/router/chain_router.go similarity index 97% rename from snow/networking/router/subnet_router.go rename to snow/networking/router/chain_router.go index 006d220..4505bec 100644 --- a/snow/networking/router/subnet_router.go +++ b/snow/networking/router/chain_router.go @@ -185,7 +185,7 @@ func (sr *ChainRouter) GetAcceptedFailed(validatorID ids.ShortID, chainID ids.ID return } } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("GetAcceptedFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } sr.timeouts.Cancel(validatorID, chainID, requestID) } @@ -200,7 +200,7 @@ func (sr *ChainRouter) GetAncestors(validatorID ids.ShortID, chainID ids.ID, req if chain, exists := sr.chains[chainID.Key()]; exists { chain.GetAncestors(validatorID, requestID, containerID) } else { - sr.log.Error("GetAcceptedFailed(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) + sr.log.Debug("GetAncestors(%s, %s, %d) dropped due to unknown chain", validatorID, chainID, requestID) } } @@ -217,7 +217,7 @@ func (sr *ChainRouter) MultiPut(validatorID ids.ShortID, chainID ids.ID, request sr.timeouts.Cancel(validatorID, chainID, requestID) } } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Debug("MultiPut(%s, %s, %d, %d) dropped due to unknown chain", validatorID, chainID, requestID, len(containers)) } } @@ -236,7 +236,7 @@ func (sr *ChainRouter) GetAncestorsFailed(validatorID ids.ShortID, chainID ids.I return } } else { - sr.log.Debug("message referenced a chain, %s, this node doesn't validate", chainID) + sr.log.Error("GetAncestorsFailed(%s, %s, %d, %d) dropped due to unknown chain", validatorID, chainID, requestID) } sr.timeouts.Cancel(validatorID, chainID, requestID) } From 91852fe932ecc617449dc500430e87e9cf4af1b0 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 15 Jun 2020 17:08:25 -0400 Subject: [PATCH 39/42] nit --- api/keystore/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 6481419..1efaafb 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -36,8 +36,8 @@ const ( // // As per issue https://github.com/ava-labs/gecko/issues/195 it was found // the longer the length of password the slower zxcvbn.PasswordStrength() - // performs. To avoid performance issues and a DoS vector, we only check the - // first 50 characters of the password. + // performs. To avoid performance issues, and a DoS vector, we only check + // the first 50 characters of the password. maxCheckedPassLen = 50 // requiredPassScore defines the score a password must achieve to be From ec953d6ec38c3658a596bee1807d11e58349ddfc Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Mon, 15 Jun 2020 15:08:03 -0700 Subject: [PATCH 40/42] Fix log level. --- api/keystore/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 997b223..14ae57e 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -146,7 +146,7 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Verbo("CreateUser called with %.*s", maxUserPassLen, args.Username) + ks.log.Debug("CreateUser called with %.*s", maxUserPassLen, args.Username) if err := ks.AddUser(args.Username, args.Password); err != nil { return err } From b950f016d884564b89df62e4f9996a79fb6a5095 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Mon, 15 Jun 2020 21:53:29 -0400 Subject: [PATCH 41/42] Fix platform bootstrapped function to initialize fx --- vms/platformvm/vm.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go index 8d5a71c..2c71593 100644 --- a/vms/platformvm/vm.go +++ b/vms/platformvm/vm.go @@ -405,10 +405,14 @@ func (vm *VM) createChain(tx *CreateChainTx) { } // Bootstrapping marks this VM as bootstrapping -func (vm *VM) Bootstrapping() error { return nil } +func (vm *VM) Bootstrapping() error { + return vm.fx.Bootstrapping() +} // Bootstrapped marks this VM as bootstrapped -func (vm *VM) Bootstrapped() error { return nil } +func (vm *VM) Bootstrapped() error { + return vm.fx.Bootstrapped() +} // Shutdown this blockchain func (vm *VM) Shutdown() error { From aa5422696e5ee5620d0fd827385cec1ad3bfd8f8 Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Mon, 15 Jun 2020 21:14:54 -0700 Subject: [PATCH 42/42] Set helpers to Debug log level. --- api/keystore/service.go | 2 +- vms/platformvm/service.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/keystore/service.go b/api/keystore/service.go index 14ae57e..ec48d48 100644 --- a/api/keystore/service.go +++ b/api/keystore/service.go @@ -146,7 +146,7 @@ func (ks *Keystore) CreateUser(_ *http.Request, args *CreateUserArgs, reply *Cre ks.lock.Lock() defer ks.lock.Unlock() - ks.log.Debug("CreateUser called with %.*s", maxUserPassLen, args.Username) + ks.log.Info("Keystore: CreateUser called with %.*s", maxUserPassLen, args.Username) if err := ks.AddUser(args.Username, args.Password); err != nil { return err } diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 2be9e12..9913608 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -915,7 +915,7 @@ func (service *Service) Sign(_ *http.Request, args *SignArgs, reply *SignRespons // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Info("Platform: signAddDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("signAddDefaultSubnetValidatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetValidatorTx) @@ -938,7 +938,7 @@ func (service *Service) signAddDefaultSubnetValidatorTx(tx *addDefaultSubnetVali // Sign [unsigned] with [key] func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDelegatorTx, key *crypto.PrivateKeySECP256K1R) (*addDefaultSubnetDelegatorTx, error) { - service.vm.Ctx.Log.Info("Platform: signAddDefaultSubnetDelegatorTx called") + service.vm.Ctx.Log.Debug("signAddDefaultSubnetDelegatorTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedAddDefaultSubnetDelegatorTx) @@ -961,7 +961,7 @@ func (service *Service) signAddDefaultSubnetDelegatorTx(tx *addDefaultSubnetDele // Sign [xt] with [key] func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.PrivateKeySECP256K1R) (*CreateSubnetTx, error) { - service.vm.Ctx.Log.Info("Platform: signCreateSubnetTx called") + service.vm.Ctx.Log.Debug("signCreateSubnetTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedCreateSubnetTx) @@ -984,7 +984,7 @@ func (service *Service) signCreateSubnetTx(tx *CreateSubnetTx, key *crypto.Priva // Sign [tx] with [key] func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256K1R) (*ExportTx, error) { - service.vm.Ctx.Log.Info("Platform: signExportTx called") + service.vm.Ctx.Log.Debug("signExportTx called") // TODO: Should we check if tx is already signed? unsignedIntf := interface{}(&tx.UnsignedExportTx) @@ -1012,7 +1012,7 @@ func (service *Service) signExportTx(tx *ExportTx, key *crypto.PrivateKeySECP256 // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signAddNonDefaultSubnetValidatorTx(tx *addNonDefaultSubnetValidatorTx, key *crypto.PrivateKeySECP256K1R) (*addNonDefaultSubnetValidatorTx, error) { - service.vm.Ctx.Log.Info("Platform: signAddNonDefaultSubnetValidatorTx called") + service.vm.Ctx.Log.Debug("signAddNonDefaultSubnetValidatorTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedAddNonDefaultSubnetValidatorTx) @@ -1203,7 +1203,7 @@ func (service *Service) ImportAVA(_ *http.Request, args *ImportAVAArgs, response // Sorts tx.ControlSigs before returning // Assumes each element of tx.ControlSigs is actually a signature, not just empty bytes func (service *Service) signCreateChainTx(tx *CreateChainTx, key *crypto.PrivateKeySECP256K1R) (*CreateChainTx, error) { - service.vm.Ctx.Log.Info("Platform: signCreateChainTx called") + service.vm.Ctx.Log.Debug("signCreateChainTx called") // Compute the byte repr. of the unsigned tx and the signature of [key] over it unsignedIntf := interface{}(&tx.UnsignedCreateChainTx)