Skip to content

Commit

Permalink
Update OpenTelemetry and use an internal package
Browse files Browse the repository at this point in the history
The "autoexport" and "autoprop" packages import and configure standard
exporters according to the current Specification, 1.39.0. The correct
correlation of logs to trace context uses underscores now.

The "google.golang.org/grpc" module raised its required Go version to
1.22.7 this release but has since reverted that change.

The "spancheck" linter reminds us to call "Span.End" after calling "tracing.Start".

Issue: PGO-1954
See: https://opentelemetry.io/docs/specs/otel
See: https://opentelemetry.io/docs/specs/otel/compatibility/logging_trace_context
See: https://www.github.com/open-telemetry/opentelemetry-go/issues/5969
  • Loading branch information
cbandy committed Dec 10, 2024
1 parent 11510f1 commit 7d4d44e
Show file tree
Hide file tree
Showing 23 changed files with 468 additions and 207 deletions.
10 changes: 10 additions & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,13 @@ linters-settings:
depguard:
rules:
everything:
list-mode: lax
allow:
- go.opentelemetry.io/otel/semconv/v1.27.0
deny:
- pkg: go.opentelemetry.io/otel/semconv
desc: Use "go.opentelemetry.io/otel/semconv/v1.27.0" instead.

- pkg: io/ioutil
desc: >
Use the "io" and "os" packages instead.
Expand Down Expand Up @@ -93,6 +99,10 @@ linters-settings:
alias: apierrors
no-unaliased: true

spancheck:
extra-start-span-signatures:
- 'github.com/crunchydata/postgres-operator/internal/tracing.Start:opentelemetry'

issues:
exclude-generated: strict
exclude-rules:
Expand Down
15 changes: 8 additions & 7 deletions cmd/postgres-operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package main

import (
"context"
"errors"
"fmt"
"net/http"
"os"
Expand All @@ -16,7 +17,6 @@ import (
"time"
"unicode"

"go.opentelemetry.io/otel"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/healthz"
Expand All @@ -33,12 +33,11 @@ import (
"github.com/crunchydata/postgres-operator/internal/logging"
"github.com/crunchydata/postgres-operator/internal/naming"
"github.com/crunchydata/postgres-operator/internal/registration"
"github.com/crunchydata/postgres-operator/internal/tracing"
"github.com/crunchydata/postgres-operator/internal/upgradecheck"
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
)

var versionString string

// assertNoError panics when err is not nil.
func assertNoError(err error) {
if err != nil {
Expand Down Expand Up @@ -125,6 +124,7 @@ func main() {
running, stopRunning := context.WithCancel(context.Background())
defer stopRunning()

initVersion()
initLogging()
log := logging.FromContext(running)
log.V(1).Info("debug flag set to true")
Expand Down Expand Up @@ -159,11 +159,14 @@ func main() {
// Initialize OpenTelemetry and flush data when there is a panic.
otelFinish, err := initOpenTelemetry(running)
assertNoError(err)
defer otelFinish(running)
defer func(ctx context.Context) { _ = otelFinish(ctx) }(running)

tracing.SetDefaultTracer(tracing.New("github.com/CrunchyData/postgres-operator"))

cfg, err := runtime.GetConfig()
assertNoError(err)

cfg.UserAgent = userAgent
cfg.Wrap(otelTransportWrapper())

// TODO(controller-runtime): Set config.WarningHandler instead after v0.19.0.
Expand Down Expand Up @@ -250,8 +253,7 @@ func main() {
}

// Flush any telemetry with the remaining time we have.
otelFinish(stopping)
if err != nil {
if err = errors.Join(err, otelFinish(stopping)); err != nil {
log.Error(err, "shutdown failed")
} else {
log.Info("shutdown complete")
Expand All @@ -266,7 +268,6 @@ func addControllersToManager(mgr runtime.Manager, log logging.Logger, reg regist
Owner: postgrescluster.ControllerName,
Recorder: mgr.GetEventRecorderFor(postgrescluster.ControllerName),
Registration: reg,
Tracer: otel.Tracer(postgrescluster.ControllerName),
}

if err := pgReconciler.SetupWithManager(mgr); err != nil {
Expand Down
118 changes: 65 additions & 53 deletions cmd/postgres-operator/open_telemetry.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,79 +6,91 @@ package main

import (
"context"
"fmt"
"io"
"errors"
"net/http"
"os"

"go.opentelemetry.io/contrib/exporters/autoexport"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/contrib/propagators/autoprop"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/sdk/trace"
)
semconv "go.opentelemetry.io/otel/semconv/v1.27.0"

func initOpenTelemetry(ctx context.Context) (func(context.Context), error) {
// At the time of this writing, the SDK (go.opentelemetry.io/[email protected])
// does not automatically initialize any exporter. We import the OTLP and
// stdout exporters and configure them below. Much of the OTLP exporter can
// be configured through environment variables.
//
// - https://github.com/open-telemetry/opentelemetry-go/issues/2310
// - https://github.com/open-telemetry/opentelemetry-specification/blob/v1.8.0/specification/sdk-environment-variables.md
"github.com/crunchydata/postgres-operator/internal/logging"
)

switch os.Getenv("OTEL_TRACES_EXPORTER") {
case "json":
var closer io.Closer
filename := os.Getenv("OTEL_JSON_FILE")
options := []stdouttrace.Option{}
func initOpenTelemetry(ctx context.Context) (func(context.Context) error, error) {
var started []interface{ Shutdown(context.Context) error }

if filename != "" {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return nil, fmt.Errorf("unable to open exporter file: %w", err)
}
closer = file
options = append(options, stdouttrace.WithWriter(file))
// shutdown returns the results of calling all the Shutdown methods in started.
var shutdown = func(ctx context.Context) error {
var err error
for _, s := range started {
err = errors.Join(err, s.Shutdown(ctx))
}
started = nil
return err
}

exporter, err := stdouttrace.New(options...)
if err != nil {
return nil, fmt.Errorf("unable to initialize stdout exporter: %w", err)
}
// The default for OTEL_PROPAGATORS is "tracecontext,baggage".
otel.SetTextMapPropagator(autoprop.NewTextMapPropagator())

provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
flush := func(ctx context.Context) {
_ = provider.Shutdown(ctx)
if closer != nil {
_ = closer.Close()
}
}
// Skip any remaining setup when OTEL_SDK_DISABLED is exactly "true".
// - https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables
if os.Getenv("OTEL_SDK_DISABLED") == "true" {
return shutdown, nil
}

otel.SetTracerProvider(provider)
return flush, nil
log := logging.FromContext(ctx).WithName("open-telemetry")
otel.SetLogger(log)
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
// TODO(events): Emit this as an event instead.
log.V(1).Info(semconv.ExceptionEventName,
string(semconv.ExceptionMessageKey), err)
}))

case "otlp":
client := otlptracehttp.NewClient()
exporter, err := otlptrace.New(context.TODO(), client)
if err != nil {
return nil, fmt.Errorf("unable to initialize OTLP exporter: %w", err)
}
// Build a resource from the OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables.
// - https://opentelemetry.io/docs/languages/go/resources
self, _ := resource.Merge(resource.NewSchemaless(
semconv.ServiceVersion(versionString),
), resource.Default())

provider := trace.NewTracerProvider(trace.WithBatcher(exporter))
flush := func(ctx context.Context) {
_ = provider.Shutdown(ctx)
// Provide defaults for some other detectable attributes.
if r, err := resource.New(ctx,
resource.WithProcessRuntimeName(),
resource.WithProcessRuntimeVersion(),
resource.WithProcessRuntimeDescription(),
); err == nil {
self, _ = resource.Merge(r, self)
}
if r, err := resource.New(ctx,
resource.WithHost(),
resource.WithOS(),
); err == nil {
self, _ = resource.Merge(r, self)
}

// The default for OTEL_TRACES_EXPORTER is "otlp" but we prefer "none".
// Only assign an exporter when the environment variable is set.
if os.Getenv("OTEL_TRACES_EXPORTER") != "" {
exporter, err := autoexport.NewSpanExporter(ctx)
if err != nil {
return nil, errors.Join(err, shutdown(ctx))
}

// The defaults for this batch processor come from the OTEL_BSP_* environment variables.
// - https://pkg.go.dev/go.opentelemetry.io/otel/sdk/internal/env
provider := trace.NewTracerProvider(
trace.WithBatcher(exporter),
trace.WithResource(self),
)
started = append(started, provider)
otel.SetTracerProvider(provider)
return flush, nil
}

// $OTEL_TRACES_EXPORTER is unset or unknown, so no TracerProvider has been assigned.
// The default at this time is a single "no-op" tracer.

return func(context.Context) {}, nil
return shutdown, nil
}

// otelTransportWrapper creates a function that wraps the provided net/http.RoundTripper
Expand Down
26 changes: 26 additions & 0 deletions cmd/postgres-operator/version.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2017 - 2024 Crunchy Data Solutions, Inc.
//
// SPDX-License-Identifier: Apache-2.0

package main

import (
"fmt"
"os"
"path/filepath"
"runtime"
)

var userAgent string
var versionString string

func initVersion() {
command := "unknown"
if len(os.Args) > 0 && len(os.Args[0]) > 0 {
command = filepath.Base(os.Args[0])
}
if len(versionString) > 0 {
command += "/" + versionString
}
userAgent = fmt.Sprintf("%s (%s/%s)", command, runtime.GOOS, runtime.GOARCH)
}
66 changes: 43 additions & 23 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/crunchydata/postgres-operator

go 1.22.0
go 1.22.7

require (
github.com/go-logr/logr v1.4.2
Expand All @@ -14,14 +14,13 @@ require (
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.9.3
github.com/xdg-go/stringprep v1.0.2
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
golang.org/x/crypto v0.27.0
go.opentelemetry.io/contrib/exporters/autoexport v0.57.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
go.opentelemetry.io/contrib/propagators/autoprop v0.57.0
go.opentelemetry.io/otel v1.32.0
go.opentelemetry.io/otel/sdk v1.32.0
go.opentelemetry.io/otel/trace v1.32.0
golang.org/x/crypto v0.28.0
golang.org/x/tools v0.22.0
gotest.tools/v3 v3.1.0
k8s.io/api v0.31.0
Expand Down Expand Up @@ -54,38 +53,59 @@ require (
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/spdystream v0.4.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/common v0.60.1 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect
go.opentelemetry.io/contrib/propagators/aws v1.32.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect
go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect
go.opentelemetry.io/contrib/propagators/ot v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.54.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 // indirect
go.opentelemetry.io/otel/log v0.8.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/sdk/log v0.8.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.9.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/term v0.25.0 // indirect
golang.org/x/text v0.20.0 // indirect
golang.org/x/time v0.5.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.66.2 // indirect
google.golang.org/protobuf v1.34.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/grpc v1.68.0 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading

0 comments on commit 7d4d44e

Please sign in to comment.