Merge pull request #69 from asymmetric-research/running-updates

WIP: Running updates

.gitignore

@@ -1,4 +1,4 @@
-# for prometheus configs:
+# for prometheus:
 .prometheus
 
 # builds:

README.md

@@ -48,7 +48,8 @@ The exporter is configured via the following command line arguments:
 
 ### Notes on Configuration
 
-* `-light-mode` is incompatible with both `-monitor-block-sizes` and `-comprehensive-slot-tracking`.
+* `-light-mode` is incompatible with `-nodekey`, `-balance-address`, `-monitor-block-sizes`, and 
+`-comprehensive-slot-tracking`, as these options control metrics which are not monitored in `-light-mode`.
 * ***WARNING***:
   * Configuring `-comprehensive-slot-tracking` will lead to potentially thousands of new Prometheus metrics being 
   created every epoch.
@@ -75,17 +76,17 @@ The table below describes all the metrics collected by the `solana_exporter`:
 | `solana_node_num_slots_behind`      | The number of slots that the node is behind the latest cluster confirmed slot.*          | N/A                           |
 | `solana_node_minimum_ledger_slot`   | The lowest slot that the node has information about in its ledger.*                      | N/A                           |
 | `solana_node_first_available_block` | The slot of the lowest confirmed block that has not been purged from the node's ledger.* | N/A                           |
-| `solana_total_transactions`         | Total number of transactions processed without error since genesis.*                     | N/A                           |
+| `solana_node_total_transactions`    | Total number of transactions processed without error since genesis.*                     | N/A                           |
-| `solana_slot_height`                | The current slot number.*                                                                | N/A                           |
+| `solana_node_slot_height`           | The current slot number.*                                                                | N/A                           |
-| `solana_epoch_number`               | The current epoch number.*                                                               | N/A                           |
+| `solana_node_epoch_number`          | The current epoch number.*                                                               | N/A                           |
-| `solana_epoch_first_slot`           | Current epoch's first slot \[inclusive\].*                                               | N/A                           |
+| `solana_node_epoch_first_slot`      | Current epoch's first slot \[inclusive\].*                                               | N/A                           |
-| `solana_epoch_last_slot`            | Current epoch's last slot \[inclusive\].*                                                | N/A                           |
+| `solana_node_epoch_last_slot`       | Current epoch's last slot \[inclusive\].*                                                | N/A                           |
-| `solana_leader_slots`               | Number of slots processed.                                                               | `status`, `nodekey`           |
+| `solana_node_leader_slots`          | Number of slots processed.                                                               | `status`, `nodekey`           |
-| `solana_leader_slots_by_epoch`      | Number of slots processed.                                                               | `status`, `nodekey`, `epoch`  |
+| `solana_node_leader_slots_by_epoch` | Number of slots processed.                                                               | `status`, `nodekey`, `epoch`  |
-| `solana_inflation_rewards`          | Inflation reward earned.                                                                 | `votekey`, `epoch`            |
+| `solana_node_inflation_rewards`     | Inflation reward earned.                                                                 | `votekey`, `epoch`            |
-| `solana_fee_rewards`                | Transaction fee rewards earned.                                                          | `nodekey`, `epoch`            |
+| `solana_node_fee_rewards`           | Transaction fee rewards earned.                                                          | `nodekey`, `epoch`            |
-| `solana_block_size`                 | Number of transactions per block.*                                                       | `nodekey`, `transaction_type` |
+| `solana_node_block_size`            | Number of transactions per block.                                                        | `nodekey`, `transaction_type` |
-| `solana_block_height`               | The current block height of the node.                                                    | N/A                           |
+| `solana_node_block_height`          | The current block height of the node.*                                                   | N/A                           |
 
 ***NOTE***: An `*` in the description indicates that the metric **is** tracked in `-light-mode`.
 

cmd/solana_exporter/config.go

@@ -59,8 +59,22 @@ func NewExporterConfig(
 		"monitorBlockSizes", monitorBlockSizes,
 		"lightMode", lightMode,
 	)
-	if lightMode && (monitorBlockSizes || comprehensiveSlotTracking) {
+	if lightMode {
-		return nil, fmt.Errorf("-light-mode is not compatible with -comprehensiveSlotTracking or -monitorBlockSizes")
+		if comprehensiveSlotTracking {
+			return nil, fmt.Errorf("'-light-mode' is imcompatible with `-comprehensive-slot-tracking`")
+		}
+
+		if monitorBlockSizes {
+			return nil, fmt.Errorf("'-light-mode' is imcompatible with `-monitor-block-sizes`")
+		}
+
+		if len(nodeKeys) > 0 {
+			return nil, fmt.Errorf("'-light-mode' is imcompatible with `-nodekey`")
+		}
+
+		if len(balanceAddresses) > 0 {
+			return nil, fmt.Errorf("'-light-mode' is imcompatible with `-balance-addresses`")
+		}
 	}
 
 	// get votekeys from rpc:

cmd/solana_exporter/config_test.go

@@ -44,7 +44,7 @@ func TestNewExporterConfig(t *testing.T) {
 			listenAddress:             ":8080",
 			nodeKeys:                  simulator.Nodekeys,
 			balanceAddresses:          []string{"xxx", "yyy", "zzz"},
-			comprehensiveSlotTracking: true,
+			comprehensiveSlotTracking: false,
 			monitorBlockSizes:         false,
 			lightMode:                 true,
 			slotPace:                  time.Second,

cmd/solana_exporter/exporter.go

@@ -55,7 +55,7 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle
 		config:    config,
 		ValidatorActiveStake: NewGaugeDesc(
 			"solana_validator_active_stake",
-			fmt.Sprintf("Active stake per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
+			fmt.Sprintf("Active stake (in SOL) per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
 			VotekeyLabel, NodekeyLabel,
 		),
 		ValidatorLastVote: NewGaugeDesc(
@@ -134,7 +134,7 @@ func (c *SolanaCollector) collectVoteAccounts(ctx context.Context, ch chan<- pro
 
 	for _, account := range append(voteAccounts.Current, voteAccounts.Delinquent...) {
 		accounts := []string{account.VotePubkey, account.NodePubkey}
-		ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake), accounts...)
+		ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake)/rpc.LamportsInSol, accounts...)
 		ch <- c.ValidatorLastVote.MustNewConstMetric(float64(account.LastVote), accounts...)
 		ch <- c.ValidatorRootSlot.MustNewConstMetric(float64(account.RootSlot), accounts...)
 	}

cmd/solana_exporter/exporter_test.go

@@ -210,11 +210,13 @@ func TestSolanaCollector(t *testing.T) {
 	collector := NewSolanaCollector(client, newTestConfig(simulator, false))
 	prometheus.NewPedanticRegistry().MustRegister(collector)
 
+	stake := float64(1_000_000) / rpc.LamportsInSol
+
 	testCases := []collectionTest{
 		collector.ValidatorActiveStake.makeCollectionTest(
-			NewLV(1_000_000, "aaa", "AAA"),
+			NewLV(stake, "aaa", "AAA"),
-			NewLV(1_000_000, "bbb", "BBB"),
+			NewLV(stake, "bbb", "BBB"),
-			NewLV(1_000_000, "ccc", "CCC"),
+			NewLV(stake, "ccc", "CCC"),
 		),
 		collector.ValidatorLastVote.makeCollectionTest(
 			NewLV(34, "aaa", "AAA"),

cmd/solana_exporter/slots.go

@@ -39,7 +39,7 @@ type SlotWatcher struct {
 	EpochLastSlotMetric      prometheus.Gauge
 	LeaderSlotsMetric        *prometheus.CounterVec
 	LeaderSlotsByEpochMetric *prometheus.CounterVec
-	InflationRewardsMetric   *prometheus.GaugeVec
+	InflationRewardsMetric   *prometheus.CounterVec
 	FeeRewardsMetric         *prometheus.CounterVec
 	BlockSizeMetric          *prometheus.GaugeVec
 	BlockHeightMetric        prometheus.Gauge
@@ -53,28 +53,30 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher {
 		config: config,
 		// metrics:
 		TotalTransactionsMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_total_transactions",
+			// even though this isn't a counter, it is supposed to act as one,
+			// and so we name it with the _total suffix
+			Name: "solana_node_transactions_total",
 			Help: "Total number of transactions processed without error since genesis.",
 		}),
 		SlotHeightMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_slot_height",
+			Name: "solana_node_slot_height",
 			Help: "The current slot number",
 		}),
 		EpochNumberMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_epoch_number",
+			Name: "solana_node_epoch_number",
 			Help: "The current epoch number.",
 		}),
 		EpochFirstSlotMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_epoch_first_slot",
+			Name: "solana_node_epoch_first_slot",
 			Help: "Current epoch's first slot [inclusive].",
 		}),
 		EpochLastSlotMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_epoch_last_slot",
+			Name: "solana_node_epoch_last_slot",
 			Help: "Current epoch's last slot [inclusive].",
 		}),
 		LeaderSlotsMetric: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
-				Name: "solana_leader_slots",
+				Name: "solana_validator_leader_slots_total",
 				Help: fmt.Sprintf(
 					"Number of slots processed, grouped by %s, and %s ('%s' or '%s')",
 					NodekeyLabel, SkipStatusLabel, StatusValid, StatusSkipped,
@@ -84,7 +86,7 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher {
 		),
 		LeaderSlotsByEpochMetric: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
-				Name: "solana_leader_slots_by_epoch",
+				Name: "solana_validator_leader_slots_by_epoch_total",
 				Help: fmt.Sprintf(
 					"Number of slots processed, grouped by %s, %s ('%s' or '%s'), and %s",
 					NodekeyLabel, SkipStatusLabel, StatusValid, StatusSkipped, EpochLabel,
@@ -92,29 +94,29 @@ func NewSlotWatcher(client *rpc.Client, config *ExporterConfig) *SlotWatcher {
 			},
 			[]string{NodekeyLabel, EpochLabel, SkipStatusLabel},
 		),
-		InflationRewardsMetric: prometheus.NewGaugeVec(
+		InflationRewardsMetric: prometheus.NewCounterVec(
-			prometheus.GaugeOpts{
+			prometheus.CounterOpts{
-				Name: "solana_inflation_rewards",
+				Name: "solana_validator_inflation_rewards_total",
 				Help: fmt.Sprintf("Inflation reward earned, grouped by %s and %s", VotekeyLabel, EpochLabel),
 			},
 			[]string{VotekeyLabel, EpochLabel},
 		),
 		FeeRewardsMetric: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
-				Name: "solana_fee_rewards",
+				Name: "solana_validator_fee_rewards_total",
 				Help: fmt.Sprintf("Transaction fee rewards earned, grouped by %s and %s", NodekeyLabel, EpochLabel),
 			},
 			[]string{NodekeyLabel, EpochLabel},
 		),
 		BlockSizeMetric: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
-				Name: "solana_block_size",
+				Name: "solana_validator_block_size",
 				Help: fmt.Sprintf("Number of transactions per block, grouped by %s", NodekeyLabel),
 			},
 			[]string{NodekeyLabel, TransactionTypeLabel},
 		),
 		BlockHeightMetric: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "solana_block_height",
+			Name: "solana_node_block_height",
 			Help: "The current block height of the node",
 		}),
 	}
@@ -174,6 +176,7 @@ func (c *SlotWatcher) WatchSlots(ctx context.Context) {
 				c.trackEpoch(ctx, epochInfo)
 			}
 
+			c.logger.Infof("Current slot: %v", epochInfo.AbsoluteSlot)
 			c.TotalTransactionsMetric.Set(float64(epochInfo.TransactionCount))
 			c.SlotHeightMetric.Set(float64(epochInfo.AbsoluteSlot))
 			c.BlockHeightMetric.Set(float64(epochInfo.BlockHeight))
@@ -281,21 +284,21 @@ func (c *SlotWatcher) checkValidSlotRange(from, to int64) error {
 
 // moveSlotWatermark performs all the slot-watching tasks required to move the slotWatermark to the provided 'to' slot.
 func (c *SlotWatcher) moveSlotWatermark(ctx context.Context, to int64) {
-	c.fetchAndEmitBlockProduction(ctx, to)
+	c.logger.Infof("Moving watermark %v -> %v", c.slotWatermark, to)
-	c.fetchAndEmitBlockInfos(ctx, to)
+	startSlot := c.slotWatermark + 1
+	c.fetchAndEmitBlockProduction(ctx, startSlot, to)
+	c.fetchAndEmitBlockInfos(ctx, startSlot, to)
 	c.slotWatermark = to
 }
 
-// fetchAndEmitBlockProduction fetches block production up to the provided endSlot, emits the prometheus metrics,
+// fetchAndEmitBlockProduction fetches block production from startSlot up to the provided endSlot [inclusive],
-// and updates the SlotWatcher.slotWatermark accordingly
+// and emits the prometheus metrics,
-func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot int64) {
+func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, startSlot, endSlot int64) {
 	if c.config.LightMode {
 		c.logger.Debug("Skipping block-production fetching in light mode.")
 		return
 	}
-	// add 1 because GetBlockProduction's range is inclusive, and the watermark is already tracked
+	c.logger.Debugf("Fetching block production in [%v -> %v]", startSlot, endSlot)
-	startSlot := c.slotWatermark + 1
-	c.logger.Infof("Fetching block production in [%v -> %v]", startSlot, endSlot)
 
 	// make sure the bounds are contained within the epoch we are currently watching:
 	if err := c.checkValidSlotRange(startSlot, endSlot); err != nil {
@@ -324,18 +327,17 @@ func (c *SlotWatcher) fetchAndEmitBlockProduction(ctx context.Context, endSlot i
 		}
 	}
 
-	c.logger.Infof("Fetched block production in [%v -> %v]", startSlot, endSlot)
+	c.logger.Debugf("Fetched block production in [%v -> %v]", startSlot, endSlot)
 }
 
 // fetchAndEmitBlockInfos fetches and emits all the fee rewards (+ block sizes) for the tracked addresses between the
-// slotWatermark and endSlot
+// startSlot and endSlot [inclusive]
-func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, endSlot int64) {
+func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, startSlot, endSlot int64) {
 	if c.config.LightMode {
 		c.logger.Debug("Skipping block-infos fetching in light mode.")
 		return
 	}
-	startSlot := c.slotWatermark + 1
+	c.logger.Debugf("Fetching fee rewards in [%v -> %v]", startSlot, endSlot)
-	c.logger.Infof("Fetching fee rewards in [%v -> %v]", startSlot, endSlot)
 
 	if err := c.checkValidSlotRange(startSlot, endSlot); err != nil {
 		c.logger.Fatalf("invalid slot range: %v", err)
@@ -355,7 +357,7 @@ func (c *SlotWatcher) fetchAndEmitBlockInfos(ctx context.Context, endSlot int64)
 		}
 	}
 
-	c.logger.Infof("Fetched fee rewards in [%v -> %v]", startSlot, endSlot)
+	c.logger.Debugf("Fetched fee rewards in [%v -> %v]", startSlot, endSlot)
 }
 
 // fetchAndEmitSingleBlockInfo fetches and emits the fee reward + block size for a single block.
@@ -423,7 +425,7 @@ func (c *SlotWatcher) fetchAndEmitInflationRewards(ctx context.Context, epoch in
 	for i, rewardInfo := range rewardInfos {
 		address := c.config.VoteKeys[i]
 		reward := float64(rewardInfo.Amount) / float64(rpc.LamportsInSol)
-		c.InflationRewardsMetric.WithLabelValues(address, toString(epoch)).Set(reward)
+		c.InflationRewardsMetric.WithLabelValues(address, toString(epoch)).Add(reward)
 	}
 	c.logger.Infof("Fetched inflation reward for epoch %v.", epoch)
 	return nil

cmd/solana_exporter/slots_test.go

@@ -109,38 +109,38 @@ func TestSlotWatcher_WatchSlots_Static(t *testing.T) {
 	time.Sleep(1 * time.Second)
 
 	type testCase struct {
+		name          string
 		expectedValue float64
-		metric        prometheus.Gauge
+		metric        prometheus.Collector
 	}
 
 	// epoch info tests:
 	firstSlot, lastSlot := GetEpochBounds(epochInfo)
 	tests := []testCase{
-		{expectedValue: float64(epochInfo.AbsoluteSlot), metric: watcher.SlotHeightMetric},
+		{"slot_height", float64(epochInfo.AbsoluteSlot), watcher.SlotHeightMetric},
-		{expectedValue: float64(epochInfo.TransactionCount), metric: watcher.TotalTransactionsMetric},
+		{"total_transactions", float64(epochInfo.TransactionCount), watcher.TotalTransactionsMetric},
-		{expectedValue: float64(epochInfo.Epoch), metric: watcher.EpochNumberMetric},
+		{"epoch_number", float64(epochInfo.Epoch), watcher.EpochNumberMetric},
-		{expectedValue: float64(firstSlot), metric: watcher.EpochFirstSlotMetric},
+		{"epoch_first_slot", float64(firstSlot), watcher.EpochFirstSlotMetric},
-		{expectedValue: float64(lastSlot), metric: watcher.EpochLastSlotMetric},
+		{"epoch_last_slot", float64(lastSlot), watcher.EpochLastSlotMetric},
 	}
 
 	// add inflation reward tests:
 	inflationRewards, err := client.GetInflationReward(ctx, rpc.CommitmentFinalized, simulator.Votekeys, 2)
 	assert.NoError(t, err)
 	for i, rewardInfo := range inflationRewards {
-		epoch := fmt.Sprintf("%v", epochInfo.Epoch)
 		tests = append(
 			tests,
 			testCase{
-				expectedValue: float64(rewardInfo.Amount) / float64(rpc.LamportsInSol),
+				fmt.Sprintf("inflation_rewards_%s", simulator.Votekeys[i]),
-				metric:        watcher.InflationRewardsMetric.WithLabelValues(simulator.Votekeys[i], epoch),
+				float64(rewardInfo.Amount) / float64(rpc.LamportsInSol),
+				watcher.InflationRewardsMetric.WithLabelValues(simulator.Votekeys[i], toString(epochInfo.Epoch)),
 			},
 		)
 	}
 
-	for _, testCase := range tests {
+	for _, test := range tests {
-		name := extractName(testCase.metric.Desc())
+		t.Run(test.name, func(t *testing.T) {
-		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, test.expectedValue, testutil.ToFloat64(test.metric))
-			assert.Equal(t, testCase.expectedValue, testutil.ToFloat64(testCase.metric))
 		})
 	}
 }