Do not request metrics while validator is syncing
Workaround for https://github.com/near/nearcore/issues/3614
This commit is contained in:
parent
3b4ccc08ea
commit
d3469ff0a2
|
@ -1,12 +1,10 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"errors"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
@ -122,92 +120,74 @@ func NewSolanaCollector(rpcAddr string) prometheus.Collector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (collector nearExporter) Describe(ch chan<- *prometheus.Desc) {
|
func (c *nearExporter) Describe(ch chan<- *prometheus.Desc) {
|
||||||
ch <- collector.totalValidatorsDesc
|
ch <- c.totalValidatorsDesc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (collector nearExporter) mustEmitMetrics(ch chan<- prometheus.Metric, response *ValidatorsResponse) {
|
func (c *nearExporter) mustEmitMetrics(ch chan<- prometheus.Metric, response *ValidatorsResponse) {
|
||||||
ch <- prometheus.MustNewConstMetric(collector.totalValidatorsDesc, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.totalValidatorsDesc, prometheus.GaugeValue,
|
||||||
float64(len(response.Result.CurrentValidators)))
|
float64(len(response.Result.CurrentValidators)))
|
||||||
ch <- prometheus.MustNewConstMetric(collector.epochStartHeight, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.epochStartHeight, prometheus.GaugeValue,
|
||||||
float64(response.Result.EpochStartHeight))
|
float64(response.Result.EpochStartHeight))
|
||||||
|
|
||||||
for _, validator := range response.Result.CurrentValidators {
|
for _, validator := range response.Result.CurrentValidators {
|
||||||
stake, err := strconv.ParseFloat(validator.Stake, 64)
|
stake, err := strconv.ParseFloat(validator.Stake, 64)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch <- prometheus.NewInvalidMetric(collector.validatorStake, fmt.Errorf("invalid stake: %s", validator.Stake))
|
ch <- prometheus.NewInvalidMetric(c.validatorStake, fmt.Errorf("invalid stake: %s", validator.Stake))
|
||||||
} else {
|
} else {
|
||||||
ch <- prometheus.MustNewConstMetric(collector.validatorStake, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.validatorStake, prometheus.GaugeValue,
|
||||||
stake, validator.AccountID)
|
stake, validator.AccountID)
|
||||||
}
|
}
|
||||||
|
|
||||||
ch <- prometheus.MustNewConstMetric(collector.validatorExpectedBlocks, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.validatorExpectedBlocks, prometheus.GaugeValue,
|
||||||
float64(validator.NumExpectedBlocks), validator.AccountID)
|
float64(validator.NumExpectedBlocks), validator.AccountID)
|
||||||
ch <- prometheus.MustNewConstMetric(collector.validatorProducedBlocks, prometheus.GaugeValue,
|
ch <- prometheus.MustNewConstMetric(c.validatorProducedBlocks, prometheus.GaugeValue,
|
||||||
float64(validator.NumProducedBlocks), validator.AccountID)
|
float64(validator.NumProducedBlocks), validator.AccountID)
|
||||||
|
|
||||||
if validator.IsSlashed {
|
if validator.IsSlashed {
|
||||||
ch <- prometheus.MustNewConstMetric(collector.validatorIsSlashed, prometheus.GaugeValue, 1, validator.AccountID)
|
ch <- prometheus.MustNewConstMetric(c.validatorIsSlashed, prometheus.GaugeValue, 1, validator.AccountID)
|
||||||
} else {
|
} else {
|
||||||
ch <- prometheus.MustNewConstMetric(collector.validatorIsSlashed, prometheus.GaugeValue, 0, validator.AccountID)
|
ch <- prometheus.MustNewConstMetric(c.validatorIsSlashed, prometheus.GaugeValue, 0, validator.AccountID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (collector nearExporter) Collect(ch chan<- prometheus.Metric) {
|
func (c *nearExporter) Collect(ch chan<- prometheus.Metric) {
|
||||||
var (
|
err := c.collect(ch)
|
||||||
validatorResponse ValidatorsResponse
|
|
||||||
body []byte
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
|
|
||||||
req, err := http.NewRequest("POST", collector.rpcAddr,
|
|
||||||
bytes.NewBufferString(`{"jsonrpc":"2.0","id":1, "method":"validators", "params":[null]}`))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
ch <- prometheus.NewInvalidMetric(c.totalValidatorsDesc, err)
|
||||||
|
ch <- prometheus.NewInvalidMetric(c.epochStartHeight, err)
|
||||||
|
ch <- prometheus.NewInvalidMetric(c.validatorStake, err)
|
||||||
|
ch <- prometheus.NewInvalidMetric(c.validatorExpectedBlocks, err)
|
||||||
|
ch <- prometheus.NewInvalidMetric(c.validatorProducedBlocks, err)
|
||||||
|
ch <- prometheus.NewInvalidMetric(c.validatorIsSlashed, err)
|
||||||
}
|
}
|
||||||
req.Header.Set("content-type", "application/json")
|
}
|
||||||
|
|
||||||
resp, err := collector.client.Do(req)
|
func (c *nearExporter) collect(ch chan<- prometheus.Metric) error {
|
||||||
|
// Work around https://github.com/near/nearcore/issues/3614 by only requesting
|
||||||
|
// validator data when the node is up-to-date.
|
||||||
|
if syncing, err := c.checkSyncStatus(); err != nil {
|
||||||
|
return fmt.Errorf("checkSyncStatus: %w", err)
|
||||||
|
} else if syncing {
|
||||||
|
return errors.New("cannot export validator metrics while the node is syncing")
|
||||||
|
}
|
||||||
|
|
||||||
|
validators, err := c.getValidatorInfo()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
goto error
|
return fmt.Errorf("getValidatorInfo: %w", err)
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, err = ioutil.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
goto error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
c.mustEmitMetrics(ch, validators)
|
||||||
err = fmt.Errorf("status code %d, response: %s", resp.StatusCode, body)
|
return nil
|
||||||
goto error
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = json.Unmarshal(body, &validatorResponse); err != nil {
|
|
||||||
goto error
|
|
||||||
}
|
|
||||||
|
|
||||||
if validatorResponse.Error.Code != 0 {
|
|
||||||
err = fmt.Errorf("JSONRPC error: %s", body)
|
|
||||||
goto error
|
|
||||||
}
|
|
||||||
|
|
||||||
collector.mustEmitMetrics(ch, &validatorResponse)
|
|
||||||
return
|
|
||||||
|
|
||||||
error:
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.totalValidatorsDesc, err)
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.epochStartHeight, err)
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.validatorStake, err)
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.validatorExpectedBlocks, err)
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.validatorProducedBlocks, err)
|
|
||||||
ch <- prometheus.NewInvalidMetric(collector.validatorIsSlashed, err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
collector := NewSolanaCollector(nearRPCAddr)
|
collector := NewSolanaCollector(nearRPCAddr)
|
||||||
prometheus.MustRegister(collector)
|
prometheus.MustRegister(collector)
|
||||||
http.Handle("/metrics", promhttp.Handler())
|
http.Handle("/metrics", promhttp.Handler())
|
||||||
|
log.Print("RPC address ", nearRPCAddr)
|
||||||
|
log.Print("Listening on ", listenAddr)
|
||||||
panic(http.ListenAndServe(listenAddr, nil))
|
panic(http.ListenAndServe(listenAddr, nil))
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type NEARStatusResponse struct {
|
||||||
|
Version struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
Build string `json:"build"`
|
||||||
|
} `json:"version"`
|
||||||
|
ChainID string `json:"chain_id"`
|
||||||
|
ProtocolVersion int `json:"protocol_version"`
|
||||||
|
LatestProtocolVersion int `json:"latest_protocol_version"`
|
||||||
|
RPCAddr string `json:"rpc_addr"`
|
||||||
|
Validators []struct {
|
||||||
|
AccountID string `json:"account_id"`
|
||||||
|
IsSlashed bool `json:"is_slashed"`
|
||||||
|
} `json:"validators"`
|
||||||
|
SyncInfo struct {
|
||||||
|
LatestBlockHash string `json:"latest_block_hash"`
|
||||||
|
LatestBlockHeight int `json:"latest_block_height"`
|
||||||
|
LatestStateRoot string `json:"latest_state_root"`
|
||||||
|
LatestBlockTime time.Time `json:"latest_block_time"`
|
||||||
|
Syncing bool `json:"syncing"`
|
||||||
|
} `json:"sync_info"`
|
||||||
|
ValidatorAccountID string `json:"validator_account_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkSyncStatus determines whether the NEAR node is currently syncing the chain.
|
||||||
|
func (c *nearExporter) checkSyncStatus() (bool, error) {
|
||||||
|
r, err := http.Get(c.rpcAddr + "/status")
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("failed to request /status: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer r.Body.Close()
|
||||||
|
r.Header.Set("content-type", "application/json")
|
||||||
|
|
||||||
|
body, err := ioutil.ReadAll(r.Body)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("failed to read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.StatusCode != 200 {
|
||||||
|
return false, fmt.Errorf("status code %d, response: %s", r.StatusCode, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
var res NEARStatusResponse
|
||||||
|
if err := json.Unmarshal(body, &res); err != nil {
|
||||||
|
return false, fmt.Errorf("error decoding: %w, response: %s", err, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.SyncInfo.Syncing, nil
|
||||||
|
}
|
|
@ -0,0 +1,45 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *nearExporter) getValidatorInfo() (response *ValidatorsResponse, err error) {
|
||||||
|
req, err := http.NewRequest("POST", c.rpcAddr,
|
||||||
|
bytes.NewBufferString(`{"jsonrpc":"2.0","id":1, "method":"validators", "params":[null]}`))
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("content-type", "application/json")
|
||||||
|
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error during validators request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error reading response body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
return nil, fmt.Errorf("return status code %d, response: %s", resp.StatusCode, body)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = json.Unmarshal(body, &response); err != nil {
|
||||||
|
return nil, fmt.Errorf("error unmarshaling response body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response.Error.Code != 0 {
|
||||||
|
return nil, fmt.Errorf("JSONRPC error: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
Loading…
Reference in New Issue