132 lines
4.6 KiB
Go
132 lines
4.6 KiB
Go
// We are using promtail client version v2.8.2:
|
|
// promtail must be added using commit hashes instead of version tags, see https://github.com/grafana/loki/issues/2826
|
|
// go get github.com/grafana/loki/clients/pkg/promtail/client@9f809eda70babaf583bdf6bf335a28038f286618
|
|
|
|
package telemetry
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/blendle/zapdriver"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zapcore"
|
|
"google.golang.org/api/option"
|
|
|
|
gkzap "github.com/go-kit/kit/log/zap"
|
|
"github.com/grafana/dskit/backoff"
|
|
"github.com/grafana/dskit/flagext"
|
|
"github.com/grafana/loki/clients/pkg/promtail/api"
|
|
"github.com/grafana/loki/clients/pkg/promtail/client"
|
|
"github.com/grafana/loki/pkg/logproto"
|
|
lokiflag "github.com/grafana/loki/pkg/util/flagext"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/common/config"
|
|
"github.com/prometheus/common/model"
|
|
)
|
|
|
|
// ExternalLoggerLoki implements ExternalLogger for the Grafana Loki cloud logging.
|
|
type ExternalLoggerLoki struct {
|
|
// c is the promtail client.
|
|
c client.Client
|
|
|
|
// labels is the set of labels to be added to each log entry, based on the severity (since severity is one of the labels).
|
|
labels map[zapcore.Level]model.LabelSet
|
|
|
|
// localLogger is the zap localLogger used to log errors generated by the loki adapter. It does not use telemetry.
|
|
localLogger *zap.Logger
|
|
}
|
|
|
|
func (logger *ExternalLoggerLoki) log(time time.Time, message json.RawMessage, level zapcore.Level) {
|
|
lokiLabels := logger.labels[level]
|
|
|
|
bytes, err := message.MarshalJSON()
|
|
if err != nil {
|
|
logger.localLogger.Error("Failed to marshal log message", zap.Error(err))
|
|
return
|
|
}
|
|
entry := api.Entry{
|
|
Entry: logproto.Entry{
|
|
Timestamp: time,
|
|
Line: string(bytes),
|
|
},
|
|
|
|
Labels: lokiLabels,
|
|
}
|
|
|
|
logger.c.Chan() <- entry
|
|
}
|
|
|
|
func (logger *ExternalLoggerLoki) close() error {
|
|
logger.c.Stop()
|
|
return nil
|
|
}
|
|
|
|
// NewLokiCloudLogger creates a new Telemetry logger using Grafana Loki Cloud Logging.
|
|
// skipPrivateLogs: if set to `true`, logs with the field zap.Bool("_privateLogEntry", true) will not be logged by telemetry.
|
|
func NewLokiCloudLogger(ctx context.Context, logger *zap.Logger, url string, productName string, skipPrivateLogs bool, labels map[string]string, opts ...option.ClientOption) (*Telemetry, error) {
|
|
// The localLogger is used to log errors generated by the loki adapter. It does not use telemetry.
|
|
localLogger := logger.With(zap.String("component", "loki"))
|
|
|
|
// The gkLogger is passed into the loki client, which expects a go-kit logger.
|
|
gkLogger := gkzap.NewZapSugarLogger(localLogger, zapcore.ErrorLevel)
|
|
|
|
// Loki pegs these metrics: https://github.com/grafana/loki/blob/main/clients/pkg/promtail/client/client.go#L71-L127
|
|
m := client.NewMetrics(prometheus.DefaultRegisterer)
|
|
|
|
serverURL := flagext.URLValue{}
|
|
err := serverURL.Set(url)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse Loki client url: %v", err)
|
|
}
|
|
|
|
cfg := client.Config{
|
|
URL: serverURL,
|
|
DropRateLimitedBatches: true,
|
|
Client: config.HTTPClientConfig{},
|
|
// TenantID: We are not using the tenantID.
|
|
|
|
// Using default values from by promtail:
|
|
// https://github.com/grafana/loki/blob/bad691b5091f1ad2f09dbfb30d5395b8f57a3bcd/docs/sources/clients/promtail/configuration.md
|
|
BatchWait: 1 * time.Minute,
|
|
BatchSize: 1048576,
|
|
BackoffConfig: backoff.Config{MinBackoff: 500 * time.Millisecond, MaxBackoff: 5 * time.Minute, MaxRetries: 10},
|
|
ExternalLabels: lokiflag.LabelSet{},
|
|
Timeout: 10 * time.Second,
|
|
}
|
|
|
|
clientMaxLineSize := 1024
|
|
clientMaxLineSizeTruncate := true
|
|
|
|
c, err := client.New(m, cfg, 0, clientMaxLineSize, clientMaxLineSizeTruncate, gkLogger)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create Loki client: %v", err)
|
|
}
|
|
|
|
// Since severity is one of the labels, create a label set for each severity to avoid copying the labels map for each log entry.
|
|
lokiLabels := make(map[zapcore.Level]model.LabelSet)
|
|
for level := zapcore.DebugLevel; level <= zapcore.FatalLevel; level++ {
|
|
levLabels := model.LabelSet{}
|
|
for k, v := range labels {
|
|
levLabels[model.LabelName(k)] = model.LabelValue(v)
|
|
}
|
|
levLabels[model.LabelName("product")] = model.LabelValue(productName)
|
|
levLabels[model.LabelName("severity")] = model.LabelValue(level.CapitalString())
|
|
lokiLabels[level] = levLabels
|
|
}
|
|
|
|
return &Telemetry{
|
|
encoder: &guardianTelemetryEncoder{
|
|
Encoder: zapcore.NewJSONEncoder(zapdriver.NewProductionEncoderConfig()),
|
|
logger: &ExternalLoggerLoki{
|
|
c: c,
|
|
labels: lokiLabels,
|
|
localLogger: localLogger,
|
|
},
|
|
skipPrivateLogs: skipPrivateLogs,
|
|
},
|
|
}, nil
|
|
}
|