wormhole/node/pkg/telemetry/loki.go

// We are using promtail client version v2.8.2:
// promtail must be added using commit hashes instead of version tags, see https://github.com/grafana/loki/issues/2826
// go get github.com/grafana/loki/clients/pkg/promtail/client@9f809eda70babaf583bdf6bf335a28038f286618

package telemetry

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/blendle/zapdriver"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"google.golang.org/api/option"

	gkzap "github.com/go-kit/kit/log/zap"
	"github.com/grafana/dskit/backoff"
	"github.com/grafana/dskit/flagext"
	"github.com/grafana/loki/clients/pkg/promtail/api"
	"github.com/grafana/loki/clients/pkg/promtail/client"
	"github.com/grafana/loki/pkg/logproto"
	lokiflag "github.com/grafana/loki/pkg/util/flagext"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/config"
	"github.com/prometheus/common/model"
)

// ExternalLoggerLoki implements ExternalLogger for the Grafana Loki cloud logging.
type ExternalLoggerLoki struct {
	// c is the promtail client.
	c client.Client

	// labels is the set of labels to be added to each log entry, based on the severity (since severity is one of the labels).
	labels map[zapcore.Level]model.LabelSet

	// localLogger is the zap localLogger used to log errors generated by the loki adapter. It does not use telemetry.
	localLogger *zap.Logger
}

func (logger *ExternalLoggerLoki) log(time time.Time, message json.RawMessage, level zapcore.Level) {
	lokiLabels := logger.labels[level]

	bytes, err := message.MarshalJSON()
	if err != nil {
		logger.localLogger.Error("Failed to marshal log message", zap.Error(err))
		return
	}
	entry := api.Entry{
		Entry: logproto.Entry{
			Timestamp: time,
			Line:      string(bytes),
		},

		Labels: lokiLabels,
	}

	logger.c.Chan() <- entry
}

func (logger *ExternalLoggerLoki) close() error {
	logger.c.Stop()
	return nil
}

// NewLokiCloudLogger creates a new Telemetry logger using Grafana Loki Cloud Logging.
// skipPrivateLogs: if set to `true`, logs with the field zap.Bool("_privateLogEntry", true) will not be logged by telemetry.
func NewLokiCloudLogger(ctx context.Context, logger *zap.Logger, url string, productName string, skipPrivateLogs bool, labels map[string]string, opts ...option.ClientOption) (*Telemetry, error) {
	// The localLogger is used to log errors generated by the loki adapter. It does not use telemetry.
	localLogger := logger.With(zap.String("component", "loki"))

	// The gkLogger is passed into the loki client, which expects a go-kit logger.
	gkLogger := gkzap.NewZapSugarLogger(localLogger, zapcore.ErrorLevel)

	// Loki pegs these metrics: https://github.com/grafana/loki/blob/main/clients/pkg/promtail/client/client.go#L71-L127
	m := client.NewMetrics(prometheus.DefaultRegisterer)

	serverURL := flagext.URLValue{}
	err := serverURL.Set(url)
	if err != nil {
		return nil, fmt.Errorf("failed to parse Loki client url: %v", err)
	}

	cfg := client.Config{
		URL:                    serverURL,
		DropRateLimitedBatches: true,
		Client:                 config.HTTPClientConfig{},
		// TenantID: We are not using the tenantID.

		// Using default values from by promtail:
		// https://github.com/grafana/loki/blob/bad691b5091f1ad2f09dbfb30d5395b8f57a3bcd/docs/sources/clients/promtail/configuration.md
		BatchWait:      1 * time.Minute,
		BatchSize:      1048576,
		BackoffConfig:  backoff.Config{MinBackoff: 500 * time.Millisecond, MaxBackoff: 5 * time.Minute, MaxRetries: 10},
		ExternalLabels: lokiflag.LabelSet{},
		Timeout:        10 * time.Second,
	}

	clientMaxLineSize := 1024
	clientMaxLineSizeTruncate := true

	c, err := client.New(m, cfg, 0, clientMaxLineSize, clientMaxLineSizeTruncate, gkLogger)
	if err != nil {
		return nil, fmt.Errorf("failed to create Loki client: %v", err)
	}

	// Since severity is one of the labels, create a label set for each severity to avoid copying the labels map for each log entry.
	lokiLabels := make(map[zapcore.Level]model.LabelSet)
	for level := zapcore.DebugLevel; level <= zapcore.FatalLevel; level++ {
		levLabels := model.LabelSet{}
		for k, v := range labels {
			levLabels[model.LabelName(k)] = model.LabelValue(v)
		}
		levLabels[model.LabelName("product")] = model.LabelValue(productName)
		levLabels[model.LabelName("severity")] = model.LabelValue(level.CapitalString())
		lokiLabels[level] = levLabels
	}

	return &Telemetry{
		encoder: &guardianTelemetryEncoder{
			Encoder: zapcore.NewJSONEncoder(zapdriver.NewProductionEncoderConfig()),
			logger: &ExternalLoggerLoki{
				c:           c,
				labels:      lokiLabels,
				localLogger: localLogger,
			},
			skipPrivateLogs: skipPrivateLogs,
		},
	}, nil
}