Add distributed tracing support (#620)

* Tracing: Add tracing interceptors, add middlewares to standalone mode

* Tracing: Set otel text map propagator

* Tracing: Read tracing environment variables

* Tracing: Always log tracing status

* Tracing: Set up Opentelemetry tracer, tracing httpclient middleware

* Tracing: Initialize tracer before calling NewManager
Ensures the tracer is properly setup before instantiating the ds

* Tracing: Set version in default tracer, add TracingOpts

* Goimports

* go mod tidy

* Goimports

* Fix merge conflict

* Initialize tracing in datasource and app Manage

* Fix typo

* Update TracingMiddleware docstring

* Unexport tracingConfig

* Refactor trace provider initialization

* Add tests for httpclient tracing middleware

* Goimports

* Properly close tracing before returning from Manage

* Fix imports

* Apply default grpc middlewares in standalone mode

* Removed opentracing interceptor

* Log: Renamed arg err to error

* Remove breaking change in SetupPluginEnvironment

* Lower tracing logging to debug level

* Add DefaultTracer, add TracingMiddleware to DefaultMiddlewares

* Lint

* Lint

* Lint

* Renamed plugin env vars for tracing endpoint and propagation

* Moved DefaultTracer to backend package, removed DefaultMiddleware from middlewares stack

* Revert "Moved DefaultTracer to backend package, removed DefaultMiddleware from middlewares stack"

This reverts commit 269416d.

* Changed TracingMiddlewareName to "Tracing"

* Refactoring: Moved trace provider to internal package

* Fix typos (traceprovider -> tracerprovider, NewTextMapPropagator)

Author: xnyo

backend/app/manage.go

@@ -1,20 +1,29 @@
 package app
 
 import (
+	"fmt"
+
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
 	"github.com/grafana/grafana-plugin-sdk-go/internal/automanagement"
 )
 
 // ManageOpts can modify Manage behaviour.
 type ManageOpts struct {
 	// GRPCSettings settings for gPRC.
 	GRPCSettings backend.GRPCSettings
+
+	// TracingOpts contains settings for tracing setup.
+	TracingOpts tracing.Opts
 }
 
 // Manage starts serving the app over gPRC with automatic instance management.
 // pluginID should match the one from plugin.json.
 func Manage(pluginID string, instanceFactory InstanceFactoryFunc, opts ManageOpts) error {
-	backend.SetupPluginEnvironment(pluginID) // Enable profiler.
+	backend.SetupPluginEnvironment(pluginID)
+	if err := backend.SetupTracer(pluginID, opts.TracingOpts); err != nil {
+		return fmt.Errorf("setup tracer: %w", err)
+	}
 	handler := automanagement.NewManager(NewInstanceManager(instanceFactory))
 	return backend.Manage(pluginID, backend.ServeOpts{
 		CheckHealthHandler:  handler,

backend/datasource/manage.go

@@ -1,20 +1,29 @@
 package datasource
 
 import (
+	"fmt"
+
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
 	"github.com/grafana/grafana-plugin-sdk-go/internal/automanagement"
 )
 
 // ManageOpts can modify Manage behaviour.
 type ManageOpts struct {
 	// GRPCSettings settings for gPRC.
 	GRPCSettings backend.GRPCSettings
+
+	// TracingOpts contains settings for tracing setup.
+	TracingOpts tracing.Opts
 }
 
 // Manage starts serving the data source over gPRC with automatic instance management.
 // pluginID should match the one from plugin.json.
 func Manage(pluginID string, instanceFactory InstanceFactoryFunc, opts ManageOpts) error {
-	backend.SetupPluginEnvironment(pluginID) // Enable profiler.
+	backend.SetupPluginEnvironment(pluginID)
+	if err := backend.SetupTracer(pluginID, opts.TracingOpts); err != nil {
+		return fmt.Errorf("setup tracer: %w", err)
+	}
 	handler := automanagement.NewManager(NewInstanceManager(instanceFactory))
 	return backend.Manage(pluginID, backend.ServeOpts{
 		CheckHealthHandler:  handler,

backend/httpclient/http_client.go

@@ -193,10 +193,11 @@ type ConfigureMiddlewareFunc func(opts Options, existingMiddleware []Middleware)
 
 // DefaultMiddlewares is the default middleware applied when creating
 // new HTTP clients and no middleware is provided.
-// BasicAuthenticationMiddleware and CustomHeadersMiddleware are
+// TracingMiddleware, BasicAuthenticationMiddleware and CustomHeadersMiddleware are
 // the default middlewares.
 func DefaultMiddlewares() []Middleware {
 	return []Middleware{
+		TracingMiddleware(nil),
 		BasicAuthenticationMiddleware(),
 		CustomHeadersMiddleware(),
 		ContextualMiddleware(),

backend/httpclient/http_client_test.go

@@ -55,10 +55,11 @@ func TestNewClient(t *testing.T) {
 		require.NoError(t, err)
 		require.NotNil(t, client)
 
-		require.Len(t, usedMiddlewares, 3)
-		require.Equal(t, BasicAuthenticationMiddlewareName, usedMiddlewares[0].(MiddlewareName).MiddlewareName())
-		require.Equal(t, CustomHeadersMiddlewareName, usedMiddlewares[1].(MiddlewareName).MiddlewareName())
-		require.Equal(t, ContextualMiddlewareName, usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+		require.Len(t, usedMiddlewares, 4)
+		require.Equal(t, TracingMiddlewareName, usedMiddlewares[0].(MiddlewareName).MiddlewareName())
+		require.Equal(t, BasicAuthenticationMiddlewareName, usedMiddlewares[1].(MiddlewareName).MiddlewareName())
+		require.Equal(t, CustomHeadersMiddlewareName, usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+		require.Equal(t, ContextualMiddlewareName, usedMiddlewares[3].(MiddlewareName).MiddlewareName())
 	})
 
 	t.Run("New() with opts middleware should return expected http.Client", func(t *testing.T) {

backend/httpclient/provider_test.go

@@ -14,30 +14,33 @@ func TestProvider(t *testing.T) {
 		t.Run("Should set default middlewares", func(t *testing.T) {
 			provider := NewProvider()
 			require.NotNil(t, provider)
-			require.Equal(t, BasicAuthenticationMiddlewareName, provider.Opts.Middlewares[0].(MiddlewareName).MiddlewareName())
-			require.Equal(t, CustomHeadersMiddlewareName, provider.Opts.Middlewares[1].(MiddlewareName).MiddlewareName())
+			require.Equal(t, TracingMiddlewareName, provider.Opts.Middlewares[0].(MiddlewareName).MiddlewareName())
+			require.Equal(t, BasicAuthenticationMiddlewareName, provider.Opts.Middlewares[1].(MiddlewareName).MiddlewareName())
+			require.Equal(t, CustomHeadersMiddlewareName, provider.Opts.Middlewares[2].(MiddlewareName).MiddlewareName())
 		})
 
 		t.Run("New client should use default middlewares", func(t *testing.T) {
 			ctx := newProviderTestContext(t)
 			client, err := ctx.provider.New()
 			require.NoError(t, err)
 			require.NotNil(t, client)
-			require.Len(t, ctx.usedMiddlewares, 3)
-			require.Equal(t, BasicAuthenticationMiddlewareName, ctx.usedMiddlewares[0].(MiddlewareName).MiddlewareName())
-			require.Equal(t, CustomHeadersMiddlewareName, ctx.usedMiddlewares[1].(MiddlewareName).MiddlewareName())
-			require.Equal(t, ContextualMiddlewareName, ctx.usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+			require.Len(t, ctx.usedMiddlewares, 4)
+			require.Equal(t, TracingMiddlewareName, ctx.usedMiddlewares[0].(MiddlewareName).MiddlewareName())
+			require.Equal(t, BasicAuthenticationMiddlewareName, ctx.usedMiddlewares[1].(MiddlewareName).MiddlewareName())
+			require.Equal(t, CustomHeadersMiddlewareName, ctx.usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+			require.Equal(t, ContextualMiddlewareName, ctx.usedMiddlewares[3].(MiddlewareName).MiddlewareName())
 		})
 
 		t.Run("Transport should use default middlewares", func(t *testing.T) {
 			ctx := newProviderTestContext(t)
 			transport, err := ctx.provider.GetTransport()
 			require.NoError(t, err)
 			require.NotNil(t, transport)
-			require.Len(t, ctx.usedMiddlewares, 3)
-			require.Equal(t, BasicAuthenticationMiddlewareName, ctx.usedMiddlewares[0].(MiddlewareName).MiddlewareName())
-			require.Equal(t, CustomHeadersMiddlewareName, ctx.usedMiddlewares[1].(MiddlewareName).MiddlewareName())
-			require.Equal(t, ContextualMiddlewareName, ctx.usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+			require.Len(t, ctx.usedMiddlewares, 4)
+			require.Equal(t, TracingMiddlewareName, ctx.usedMiddlewares[0].(MiddlewareName).MiddlewareName())
+			require.Equal(t, BasicAuthenticationMiddlewareName, ctx.usedMiddlewares[1].(MiddlewareName).MiddlewareName())
+			require.Equal(t, CustomHeadersMiddlewareName, ctx.usedMiddlewares[2].(MiddlewareName).MiddlewareName())
+			require.Equal(t, ContextualMiddlewareName, ctx.usedMiddlewares[3].(MiddlewareName).MiddlewareName())
 		})
 
 		t.Run("New() with options and no middleware should return expected http client and transport", func(t *testing.T) {
@@ -78,13 +81,14 @@ func TestProvider(t *testing.T) {
 			require.Equal(t, DefaultTimeoutOptions.Timeout, client.Timeout)
 
 			t.Run("Should use configured middlewares and implement MiddlewareName", func(t *testing.T) {
-				require.Len(t, pCtx.usedMiddlewares, 6)
+				require.Len(t, pCtx.usedMiddlewares, 7)
 				require.Equal(t, "mw1", pCtx.usedMiddlewares[0].(MiddlewareName).MiddlewareName())
 				require.Equal(t, "mw2", pCtx.usedMiddlewares[1].(MiddlewareName).MiddlewareName())
 				require.Equal(t, "mw3", pCtx.usedMiddlewares[2].(MiddlewareName).MiddlewareName())
-				require.Equal(t, BasicAuthenticationMiddlewareName, pCtx.usedMiddlewares[3].(MiddlewareName).MiddlewareName())
-				require.Equal(t, CustomHeadersMiddlewareName, pCtx.usedMiddlewares[4].(MiddlewareName).MiddlewareName())
-				require.Equal(t, ContextualMiddlewareName, pCtx.usedMiddlewares[5].(MiddlewareName).MiddlewareName())
+				require.Equal(t, TracingMiddlewareName, pCtx.usedMiddlewares[3].(MiddlewareName).MiddlewareName())
+				require.Equal(t, BasicAuthenticationMiddlewareName, pCtx.usedMiddlewares[4].(MiddlewareName).MiddlewareName())
+				require.Equal(t, CustomHeadersMiddlewareName, pCtx.usedMiddlewares[5].(MiddlewareName).MiddlewareName())
+				require.Equal(t, ContextualMiddlewareName, pCtx.usedMiddlewares[6].(MiddlewareName).MiddlewareName())
 			})
 
 			t.Run("When roundtrip should call expected middlewares", func(t *testing.T) {

backend/httpclient/tracing_middleware.go

@@ -0,0 +1,65 @@
+package httpclient
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
+)
+
+const (
+	TracingMiddlewareName   = "Tracing"
+	httpContentLengthTagKey = "http.content_length"
+)
+
+// TracingMiddleware is a middleware that creates spans for each outgoing request, tracking the url, method and response
+// code as span attributes. If tracer is nil, it will use tracing.DefaultTracer().
+func TracingMiddleware(tracer trace.Tracer) Middleware {
+	return NamedMiddlewareFunc(TracingMiddlewareName, func(opts Options, next http.RoundTripper) http.RoundTripper {
+		if tracer == nil {
+			tracer = tracing.DefaultTracer()
+		}
+		return RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			ctx, span := tracer.Start(req.Context(), "HTTP Outgoing Request", trace.WithSpanKind(trace.SpanKindClient))
+			defer span.End()
+
+			req = req.WithContext(ctx)
+			for k, v := range opts.Labels {
+				span.SetAttributes(attribute.Key(k).String(v))
+			}
+
+			otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header))
+			res, err := next.RoundTrip(req)
+
+			span.SetAttributes(attribute.String("http.url", req.URL.String()))
+			span.SetAttributes(attribute.String("http.method", req.Method))
+
+			if err != nil {
+				span.RecordError(err)
+				return res, err
+			}
+
+			if res != nil {
+				// we avoid measuring contentlength less than zero because it indicates
+				// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
+				if res.ContentLength > 0 {
+					span.SetAttributes(attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength))
+				}
+
+				span.SetAttributes(attribute.Int("http.status_code", res.StatusCode))
+				if res.StatusCode >= 400 {
+					span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode)))
+				}
+			}
+
+			return res, err
+		})
+	})
+}