diff --git a/.gitignore b/.gitignore index f42e118..2fd556a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# for prometheus configs: +# for prometheus: .prometheus # builds: diff --git a/README.md b/README.md index 4a4250d..1fcc4f4 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ The exporter is configured via the following command line arguments: ### Notes on Configuration -* `-light-mode` is incompatible with both `-monitor-block-sizes` and `-comprehensive-slot-tracking`. +* `-light-mode` is incompatible with `-nodekey`, `-balance-address`, `-monitor-block-sizes`, and +`-comprehensive-slot-tracking`, as these options control metrics which are not monitored in `-light-mode`. * ***WARNING***: * Configuring `-comprehensive-slot-tracking` will lead to potentially thousands of new Prometheus metrics being created every epoch. @@ -75,17 +76,17 @@ The table below describes all the metrics collected by the `solana_exporter`: | `solana_node_num_slots_behind` | The number of slots that the node is behind the latest cluster confirmed slot.* | N/A | | `solana_node_minimum_ledger_slot` | The lowest slot that the node has information about in its ledger.* | N/A | | `solana_node_first_available_block` | The slot of the lowest confirmed block that has not been purged from the node's ledger.* | N/A | -| `solana_total_transactions` | Total number of transactions processed without error since genesis.* | N/A | -| `solana_slot_height` | The current slot number.* | N/A | -| `solana_epoch_number` | The current epoch number.* | N/A | -| `solana_epoch_first_slot` | Current epoch's first slot \[inclusive\].* | N/A | -| `solana_epoch_last_slot` | Current epoch's last slot \[inclusive\].* | N/A | -| `solana_leader_slots` | Number of slots processed. | `status`, `nodekey` | -| `solana_leader_slots_by_epoch` | Number of slots processed. | `status`, `nodekey`, `epoch` | -| `solana_inflation_rewards` | Inflation reward earned. | `votekey`, `epoch` | -| `solana_fee_rewards` | Transaction fee rewards earned. | `nodekey`, `epoch` | -| `solana_block_size` | Number of transactions per block.* | `nodekey`, `transaction_type` | -| `solana_block_height` | The current block height of the node. | N/A | +| `solana_node_total_transactions` | Total number of transactions processed without error since genesis.* | N/A | +| `solana_node_slot_height` | The current slot number.* | N/A | +| `solana_node_epoch_number` | The current epoch number.* | N/A | +| `solana_node_epoch_first_slot` | Current epoch's first slot \[inclusive\].* | N/A | +| `solana_node_epoch_last_slot` | Current epoch's last slot \[inclusive\].* | N/A | +| `solana_node_leader_slots` | Number of slots processed. | `status`, `nodekey` | +| `solana_node_leader_slots_by_epoch` | Number of slots processed. | `status`, `nodekey`, `epoch` | +| `solana_node_inflation_rewards` | Inflation reward earned. | `votekey`, `epoch` | +| `solana_node_fee_rewards` | Transaction fee rewards earned. | `nodekey`, `epoch` | +| `solana_node_block_size` | Number of transactions per block. | `nodekey`, `transaction_type` | +| `solana_node_block_height` | The current block height of the node.* | N/A | ***NOTE***: An `*` in the description indicates that the metric **is** tracked in `-light-mode`. diff --git a/cmd/solana_exporter/config.go b/cmd/solana_exporter/config.go index ca2276a..1faf621 100644 --- a/cmd/solana_exporter/config.go +++ b/cmd/solana_exporter/config.go @@ -59,8 +59,22 @@ func NewExporterConfig( "monitorBlockSizes", monitorBlockSizes, "lightMode", lightMode, ) - if lightMode && (monitorBlockSizes || comprehensiveSlotTracking) { - return nil, fmt.Errorf("-light-mode is not compatible with -comprehensiveSlotTracking or -monitorBlockSizes") + if lightMode { + if comprehensiveSlotTracking { + return nil, fmt.Errorf("'-light-mode' is imcompatible with `-comprehensive-slot-tracking`") + } + + if monitorBlockSizes { + return nil, fmt.Errorf("'-light-mode' is imcompatible with `-monitor-block-sizes`") + } + + if len(nodeKeys) > 0 { + return nil, fmt.Errorf("'-light-mode' is imcompatible with `-nodekey`") + } + + if len(balanceAddresses) > 0 { + return nil, fmt.Errorf("'-light-mode' is imcompatible with `-balance-addresses`") + } } // get votekeys from rpc: diff --git a/cmd/solana_exporter/config_test.go b/cmd/solana_exporter/config_test.go index 433a08b..520033a 100644 --- a/cmd/solana_exporter/config_test.go +++ b/cmd/solana_exporter/config_test.go @@ -44,7 +44,7 @@ func TestNewExporterConfig(t *testing.T) { listenAddress: ":8080", nodeKeys: simulator.Nodekeys, balanceAddresses: []string{"xxx", "yyy", "zzz"}, - comprehensiveSlotTracking: true, + comprehensiveSlotTracking: false, monitorBlockSizes: false, lightMode: true, slotPace: time.Second, diff --git a/cmd/solana_exporter/exporter.go b/cmd/solana_exporter/exporter.go index 12e1906..a5cc070 100644 --- a/cmd/solana_exporter/exporter.go +++ b/cmd/solana_exporter/exporter.go @@ -55,7 +55,7 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle config: config, ValidatorActiveStake: NewGaugeDesc( "solana_validator_active_stake", - fmt.Sprintf("Active stake per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel), + fmt.Sprintf("Active stake (in SOL) per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel), VotekeyLabel, NodekeyLabel, ), ValidatorLastVote: NewGaugeDesc( @@ -134,7 +134,7 @@ func (c *SolanaCollector) collectVoteAccounts(ctx context.Context, ch chan<- pro for _, account := range append(voteAccounts.Current, voteAccounts.Delinquent...) { accounts := []string{account.VotePubkey, account.NodePubkey} - ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake), accounts...) + ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake)/rpc.LamportsInSol, accounts...) ch <- c.ValidatorLastVote.MustNewConstMetric(float64(account.LastVote), accounts...) ch <- c.ValidatorRootSlot.MustNewConstMetric(float64(account.RootSlot), accounts...) } diff --git a/cmd/solana_exporter/exporter_test.go b/cmd/solana_exporter/exporter_test.go index cade12d..66b2fa5 100644 --- a/cmd/solana_exporter/exporter_test.go +++ b/cmd/solana_exporter/exporter_test.go @@ -210,11 +210,13 @@ func TestSolanaCollector(t *testing.T) { collector := NewSolanaCollector(client, newTestConfig(simulator, false)) prometheus.NewPedanticRegistry().MustRegister(collector) + stake := float64(1_000_000) / rpc.LamportsInSol + testCases := []collectionTest{ collector.ValidatorActiveStake.makeCollectionTest( - NewLV(1_000_000, "aaa", "AAA"), - NewLV(1_000_000, "bbb", "BBB"), - NewLV(1_000_000, "ccc", "CCC"), + NewLV(stake, "aaa", "AAA"), + NewLV(stake, "bbb", "BBB"), + NewLV(stake, "ccc", "CCC"), ), collector.ValidatorLastVote.makeCollectionTest( NewLV(34, "aaa", "AAA"), diff --git a/cmd/solana_exporter/slots.go b/cmd/solana_exporter/slots.go index a0df68c..1a628d4 100644 --- a/cmd/solana_exporter/slots.go +++ b/cmd/solana_exporter/slots.go @@ -39,7 +39,7 @@ type SlotWatcher struct { EpochLastSlotMetric prometheus.Gauge LeaderSlotsMetric *prometheus.CounterVec LeaderSlotsByEpochMetric *prometheus.CounterVec - InflationRewardsMetric *prometheus.GaugeVec + InflationRewardsMetric *prometheus.CounterVec FeeRewardsMetric *prometheus.CounterVec BlockSizeMetric *prometheus.GaugeVec BlockHeightMetric prometheus.Gauge @@ -53,28 +53,30 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher { config: config, // metrics: TotalTransactionsMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_total_transactions", + // even though this isn't a counter, it is supposed to act as one, + // and so we name it with the _total suffix + Name: "solana_node_transactions_total", Help: "Total number of transactions processed without error since genesis.", }), SlotHeightMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_slot_height", + Name: "solana_node_slot_height", Help: "The current slot number", }), EpochNumberMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_epoch_number", + Name: "solana_node_epoch_number", Help: "The current epoch number.", }), EpochFirstSlotMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_epoch_first_slot", + Name: "solana_node_epoch_first_slot", Help: "Current epoch's first slot [inclusive].", }), EpochLastSlotMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_epoch_last_slot", + Name: "solana_node_epoch_last_slot", Help: "Current epoch's last slot [inclusive].", }), LeaderSlotsMetric: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "solana_leader_slots", + Name: "solana_validator_leader_slots_total", Help: fmt.Sprintf( "Number of slots processed, grouped by %s, and %s ('%s' or '%s')", NodekeyLabel, SkipStatusLabel, StatusValid, StatusSkipped, @@ -84,7 +86,7 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher { ), LeaderSlotsByEpochMetric: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "solana_leader_slots_by_epoch", + Name: "solana_validator_leader_slots_by_epoch_total", Help: fmt.Sprintf( "Number of slots processed, grouped by %s, %s ('%s' or '%s'), and %s", NodekeyLabel, SkipStatusLabel, StatusValid, StatusSkipped, EpochLabel, @@ -92,29 +94,29 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher { }, []string{NodekeyLabel, EpochLabel, SkipStatusLabel}, ), - InflationRewardsMetric: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "solana_inflation_rewards", + InflationRewardsMetric: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "solana_validator_inflation_rewards_total", Help: fmt.Sprintf("Inflation reward earned, grouped by %s and %s", VotekeyLabel, EpochLabel), }, []string{VotekeyLabel, EpochLabel}, ), FeeRewardsMetric: prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "solana_fee_rewards", + Name: "solana_validator_fee_rewards_total", Help: fmt.Sprintf("Transaction fee rewards earned, grouped by %s and %s", NodekeyLabel, EpochLabel), }, []string{NodekeyLabel, EpochLabel}, ), BlockSizeMetric: prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "solana_block_size", + Name: "solana_validator_block_size", Help: fmt.Sprintf("Number of transactions per block, grouped by %s", NodekeyLabel), }, []string{NodekeyLabel, TransactionTypeLabel}, ), BlockHeightMetric: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "solana_block_height", + Name: "solana_node_block_height", Help: "The current block height of the node", }), } @@ -174,6 +176,7 @@ func (c *SlotWatcher) WatchSlots(ctx context.Context) { c.trackEpoch(ctx, epochInfo) } + c.logger.Infof("Current slot: %v", epochInfo.AbsoluteSlot) c.TotalTransactionsMetric.Set(float64(epochInfo.TransactionCount)) c.SlotHeightMetric.Set(float64(epochInfo.AbsoluteSlot)) c.BlockHeightMetric.Set(float64(epochInfo.BlockHeight)) @@ -281,21 +284,21 @@ func (c *SlotWatcher) checkValidSlotRange(from, to int64) error { // moveSlotWatermark performs all the slot-watching tasks required to move the slotWatermark to the provided 'to' slot. func (c *SlotWatcher) moveSlotWatermark(ctx context.Context, to int64) { - c.fetchAndEmitBlockProduction(ctx, to) - c.fetchAndEmitBlockInfos(ctx, to) + c.logger.Infof("Moving watermark %v -> %v", c.slotWatermark, to) + startSlot := c.slotWatermark + 1 + c.fetchAndEmitBlockProduction(ctx, startSlot, to) + c.fetchAndEmitBlockInfos(ctx, startSlot, to) c.slotWatermark = to } -// fetchAndEmitBlockProduction fetches block production up to the provided endSlot, emits the prometheus metrics, -// and updates the SlotWatcher.slotWatermark accordingly -func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot int64) { +// fetchAndEmitBlockProduction fetches block production from startSlot up to the provided endSlot [inclusive], +// and emits the prometheus metrics, +func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, startSlot, endSlot int64) { if c.config.LightMode { c.logger.Debug("Skipping block-production fetching in light mode.") return } - // add 1 because GetBlockProduction's range is inclusive, and the watermark is already tracked - startSlot := c.slotWatermark + 1 - c.logger.Infof("Fetching block production in [%v -> %v]", startSlot, endSlot) + c.logger.Debugf("Fetching block production in [%v -> %v]", startSlot, endSlot) // make sure the bounds are contained within the epoch we are currently watching: if err := c.checkValidSlotRange(startSlot, endSlot); err != nil { @@ -324,18 +327,17 @@ func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot i } } - c.logger.Infof("Fetched block production in [%v -> %v]", startSlot, endSlot) + c.logger.Debugf("Fetched block production in [%v -> %v]", startSlot, endSlot) } // fetchAndEmitBlockInfos fetches and emits all the fee rewards (+ block sizes) for the tracked addresses between the -// slotWatermark and endSlot -func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, endSlot int64) { +// startSlot and endSlot [inclusive] +func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, startSlot, endSlot int64) { if c.config.LightMode { c.logger.Debug("Skipping block-infos fetching in light mode.") return } - startSlot := c.slotWatermark + 1 - c.logger.Infof("Fetching fee rewards in [%v -> %v]", startSlot, endSlot) + c.logger.Debugf("Fetching fee rewards in [%v -> %v]", startSlot, endSlot) if err := c.checkValidSlotRange(startSlot, endSlot); err != nil { c.logger.Fatalf("invalid slot range: %v", err) @@ -355,7 +357,7 @@ func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, endSlot int64) } } - c.logger.Infof("Fetched fee rewards in [%v -> %v]", startSlot, endSlot) + c.logger.Debugf("Fetched fee rewards in [%v -> %v]", startSlot, endSlot) } // fetchAndEmitSingleBlockInfo fetches and emits the fee reward + block size for a single block. @@ -423,7 +425,7 @@ func (c *SlotWatcher) fetchAndEmitInflationRewards(ctx context.Context, epoch in for i, rewardInfo := range rewardInfos { address := c.config.VoteKeys[i] reward := float64(rewardInfo.Amount) / float64(rpc.LamportsInSol) - c.InflationRewardsMetric.WithLabelValues(address, toString(epoch)).Set(reward) + c.InflationRewardsMetric.WithLabelValues(address, toString(epoch)).Add(reward) } c.logger.Infof("Fetched inflation reward for epoch %v.", epoch) return nil diff --git a/cmd/solana_exporter/slots_test.go b/cmd/solana_exporter/slots_test.go index 687d4a1..f65629f 100644 --- a/cmd/solana_exporter/slots_test.go +++ b/cmd/solana_exporter/slots_test.go @@ -109,38 +109,38 @@ func TestSlotWatcher_WatchSlots_Static(t *testing.T) { time.Sleep(1 * time.Second) type testCase struct { + name string expectedValue float64 - metric prometheus.Gauge + metric prometheus.Collector } // epoch info tests: firstSlot, lastSlot := GetEpochBounds(epochInfo) tests := []testCase{ - {expectedValue: float64(epochInfo.AbsoluteSlot), metric: watcher.SlotHeightMetric}, - {expectedValue: float64(epochInfo.TransactionCount), metric: watcher.TotalTransactionsMetric}, - {expectedValue: float64(epochInfo.Epoch), metric: watcher.EpochNumberMetric}, - {expectedValue: float64(firstSlot), metric: watcher.EpochFirstSlotMetric}, - {expectedValue: float64(lastSlot), metric: watcher.EpochLastSlotMetric}, + {"slot_height", float64(epochInfo.AbsoluteSlot), watcher.SlotHeightMetric}, + {"total_transactions", float64(epochInfo.TransactionCount), watcher.TotalTransactionsMetric}, + {"epoch_number", float64(epochInfo.Epoch), watcher.EpochNumberMetric}, + {"epoch_first_slot", float64(firstSlot), watcher.EpochFirstSlotMetric}, + {"epoch_last_slot", float64(lastSlot), watcher.EpochLastSlotMetric}, } // add inflation reward tests: inflationRewards, err := client.GetInflationReward(ctx, rpc.CommitmentFinalized, simulator.Votekeys, 2) assert.NoError(t, err) for i, rewardInfo := range inflationRewards { - epoch := fmt.Sprintf("%v", epochInfo.Epoch) tests = append( tests, testCase{ - expectedValue: float64(rewardInfo.Amount) / float64(rpc.LamportsInSol), - metric: watcher.InflationRewardsMetric.WithLabelValues(simulator.Votekeys[i], epoch), + fmt.Sprintf("inflation_rewards_%s", simulator.Votekeys[i]), + float64(rewardInfo.Amount) / float64(rpc.LamportsInSol), + watcher.InflationRewardsMetric.WithLabelValues(simulator.Votekeys[i], toString(epochInfo.Epoch)), }, ) } - for _, testCase := range tests { - name := extractName(testCase.metric.Desc()) - t.Run(name, func(t *testing.T) { - assert.Equal(t, testCase.expectedValue, testutil.ToFloat64(testCase.metric)) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expectedValue, testutil.ToFloat64(test.metric)) }) } }