diff --git a/caddytest/integration/caddyfile_adapt/tracing.caddyfiletest b/caddytest/integration/caddyfile_adapt/tracing.caddyfiletest index 32286600122..7f88751687b 100644 --- a/caddytest/integration/caddyfile_adapt/tracing.caddyfiletest +++ b/caddytest/integration/caddyfile_adapt/tracing.caddyfiletest @@ -33,4 +33,4 @@ } } } -} \ No newline at end of file +} diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index bcbc1ff4691..e9b8c46465c 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -32,6 +32,7 @@ import ( "sync" "time" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.uber.org/zap" "go.uber.org/zap/zapcore" "golang.org/x/net/http/httpguts" @@ -810,8 +811,10 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe // do the round-trip; emit debug log with values we know are // safe, or if there is no error, emit fuller log entry + // we are wrapping the RoundTripper to make it observable start := time.Now() - res, err := h.Transport.RoundTrip(req) + otelTransport := otelhttp.NewTransport(h.Transport) + res, err := otelTransport.RoundTrip(req) duration := time.Since(start) logger := h.logger.With( zap.String("upstream", di.Upstream.String()), diff --git a/modules/caddyhttp/tracing/module.go b/modules/caddyhttp/tracing/module.go index 85fd630020e..2feaf271c04 100644 --- a/modules/caddyhttp/tracing/module.go +++ b/modules/caddyhttp/tracing/module.go @@ -25,7 +25,8 @@ func init() { type Tracing struct { // SpanName is a span name. It should follow the naming guidelines here: // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span - SpanName string `json:"span"` + SpanName string `json:"span"` + InjectServerTimingHeader bool `json:"server_timing,omitempty"` // otel implements opentelemetry related logic. otel openTelemetryWrapper @@ -46,7 +47,7 @@ func (ot *Tracing) Provision(ctx caddy.Context) error { ot.logger = ctx.Logger() var err error - ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName) + ot.otel, err = newOpenTelemetryWrapper(ctx, ot.SpanName, ot.InjectServerTimingHeader) return err } @@ -68,6 +69,7 @@ func (ot *Tracing) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyh // UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax: // // tracing { +// [injectServerTimingHeader ] // [span ] // } func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { @@ -94,12 +96,19 @@ func (ot *Tracing) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } for d.NextBlock(0) { - if dst, ok := paramsMap[d.Val()]; ok { - if err := setParameter(d, dst); err != nil { - return err + switch d.Val() { + case "server_timing": + if d.NextArg() { + ot.InjectServerTimingHeader = true + } + default: + if dst, ok := paramsMap[d.Val()]; ok { + if err := setParameter(d, dst); err != nil { + return err + } + } else { + return d.ArgErr() } - } else { - return d.ArgErr() } } return nil diff --git a/modules/caddyhttp/tracing/tracer.go b/modules/caddyhttp/tracing/tracer.go index 89c617bf4f0..5f74524fce5 100644 --- a/modules/caddyhttp/tracing/tracer.go +++ b/modules/caddyhttp/tracing/tracer.go @@ -5,6 +5,8 @@ import ( "fmt" "net/http" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/contrib/propagators/autoprop" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" @@ -37,20 +39,19 @@ type openTelemetryWrapper struct { handler http.Handler - spanName string + spanName string + injectServerTimingHeader bool } // newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration. -func newOpenTelemetryWrapper( - ctx context.Context, - spanName string, -) (openTelemetryWrapper, error) { +func newOpenTelemetryWrapper(ctx context.Context, spanName string, injectServerTimingHeader bool) (openTelemetryWrapper, error) { if spanName == "" { spanName = defaultSpanName } ot := openTelemetryWrapper{ - spanName: spanName, + injectServerTimingHeader: injectServerTimingHeader, + spanName: spanName, } version, _ := caddy.Version() @@ -64,7 +65,9 @@ func newOpenTelemetryWrapper( return ot, fmt.Errorf("creating trace exporter error: %w", err) } - ot.propagators = autoprop.NewTextMapPropagator() + prop := autoprop.NewTextMapPropagator() + otel.SetTextMapPropagator(prop) + ot.propagators = prop tracerProvider := globalTracerProvider.getTracerProvider( sdktrace.WithBatcher(traceExporter), @@ -84,15 +87,24 @@ func newOpenTelemetryWrapper( // serveHTTP injects a tracing context and call the next handler. func (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - ot.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header)) + spanCtx := trace.SpanContextFromContext(ctx) if spanCtx.IsValid() { traceID := spanCtx.TraceID().String() + spanID := spanCtx.SpanID().String() // Add a trace_id placeholder, accessible via `{http.vars.trace_id}`. caddyhttp.SetVar(ctx, "trace_id", traceID) + // Add a span_id placeholder, accessible via `{http.vars.span_id}`. + caddyhttp.SetVar(ctx, "span_id", spanID) // Add the trace id to the log fields for the request. if extra, ok := ctx.Value(caddyhttp.ExtraLogFieldsCtxKey).(*caddyhttp.ExtraLogFields); ok { extra.Add(zap.String("traceID", traceID)) + extra.Add(zap.String("spanID", spanID)) + } + + // Add the server-timing header so clients can make the connection + if ot.injectServerTimingHeader { + w.Header().Set("server-timing", fmt.Sprintf("traceparent;desc=\"00-%s-%s-%s\"", traceID, spanID, spanCtx.TraceFlags().String())) } } next := ctx.Value(nextCallCtxKey).(*nextCall) diff --git a/modules/caddyhttp/tracing/tracer_test.go b/modules/caddyhttp/tracing/tracer_test.go index 36a32ff46e0..057b77da6f1 100644 --- a/modules/caddyhttp/tracing/tracer_test.go +++ b/modules/caddyhttp/tracing/tracer_test.go @@ -14,9 +14,7 @@ func TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) { var otw openTelemetryWrapper var err error - if otw, err = newOpenTelemetryWrapper(ctx, - "", - ); err != nil { + if otw, err = newOpenTelemetryWrapper(ctx, "", false); err != nil { t.Errorf("newOpenTelemetryWrapper() error = %v", err) t.FailNow() }