Tracing: Support remote, rate-limited, and probabilistic sampling (#781)

* Initial work for remote, rate-limited, and probabilistic sampling

* go mod tidy

* sampling -> sampler

* Lint

* Fix wrong value for SamplerTypeRemote

* Unexport tracingConfig attributes

* go mod tidy

* Read service name from build info

* Add build.InfoGetter, get service name from build info

* Fix buildinfo error not being printed

* Add tests

* Add TestNewOtelSampler

* refactoring

* Use different log params for sampler type, param and remote

* Use different log params for sample remote url and service name

* better error handling for remote sampler service name

* Add comments for tracing sampling env var constants

* PR review feedback: use log.Fatalln

* Unexport tracingConfig.isEnabled

* PR review feedback: Log errors

* PR review feedback

* PR review feedback

* go mod tidy

Author: xnyo

backend/app/manage.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"fmt"
+	"log"
 	"os"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -25,7 +26,7 @@ func Manage(pluginID string, instanceFactory InstanceFactoryFunc, opts ManageOpt
 	// If we are running in build info mode, run that and exit
 	if buildinfo.InfoModeEnabled() {
 		if err := buildinfo.RunInfoMode(); err != nil {
-			os.Exit(1)
+			log.Fatalln(err)
 			return err
 		}
 		os.Exit(0)

backend/datasource/manage.go

@@ -2,6 +2,7 @@ package datasource
 
 import (
 	"fmt"
+	"log"
 	"os"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -25,7 +26,7 @@ func Manage(pluginID string, instanceFactory InstanceFactoryFunc, opts ManageOpt
 	// If we are running in build info mode, run that and exit
 	if buildinfo.InfoModeEnabled() {
 		if err := buildinfo.RunInfoMode(); err != nil {
-			os.Exit(1)
+			log.Fatalln(err)
 			return err
 		}
 		os.Exit(0)

backend/serve.go

@@ -110,7 +110,7 @@ func Serve(opts ServeOpts) error {
 }
 
 // GracefulStandaloneServe starts a gRPC server that is not managed by hashicorp.
-// The provided standalone.Args must have an Address set, or the function returns an error.
+// The provided standalone.Args must have an address set, or the function returns an error.
 // The function handles creating/cleaning up the standalone address file, and graceful GRPC server termination.
 // The function returns after the GRPC server has been terminated.
 func GracefulStandaloneServe(dsopts ServeOpts, info standalone.ServerSettings) error {

backend/setup.go

@@ -5,17 +5,19 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"strconv"
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 
+	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/tracing"
 	"github.com/grafana/grafana-plugin-sdk-go/build"
 	"github.com/grafana/grafana-plugin-sdk-go/internal/tracerprovider"
 )
 
-var (
+const (
 	// PluginProfilerEnvDeprecated is a deprecated constant for the GF_PLUGINS_PROFILER environment variable used to enable pprof.
 	PluginProfilerEnvDeprecated = "GF_PLUGINS_PROFILER"
 	// PluginProfilingEnabledEnv is a constant for the GF_PLUGIN_PROFILING_ENABLED environment variable used to enable pprof.
@@ -27,15 +29,33 @@ var (
 	PluginProfilingPortEnv = "GF_PLUGIN_PROFILING_PORT" // nolint:gosec
 
 	// PluginTracingOpenTelemetryOTLPAddressEnv is a constant for the GF_INSTANCE_OTLP_ADDRESS
-	// environment variable used to specify the OTLP Address.
+	// environment variable used to specify the OTLP address.
 	PluginTracingOpenTelemetryOTLPAddressEnv = "GF_INSTANCE_OTLP_ADDRESS" // nolint:gosec
 	// PluginTracingOpenTelemetryOTLPPropagationEnv is a constant for the GF_INSTANCE_OTLP_PROPAGATION
 	// environment variable used to specify the OTLP propagation format.
 	PluginTracingOpenTelemetryOTLPPropagationEnv = "GF_INSTANCE_OTLP_PROPAGATION"
 
+	// PluginTracingSamplerTypeEnv is a constant for the GF_INSTANCE_OTLP_SAMPLER_TYPE
+	// environment variable used to specify the OTLP sampler type.
+	PluginTracingSamplerTypeEnv = "GF_INSTANCE_OTLP_SAMPLER_TYPE"
+
+	// PluginTracingSamplerParamEnv is a constant for the GF_INSTANCE_OTLP_SAMPLER_PARAM
+	// environment variable used to specify an additional float parameter used by the OTLP sampler,
+	// depending on the type.
+	PluginTracingSamplerParamEnv = "GF_INSTANCE_OTLP_SAMPLER_PARAM"
+
+	// PluginTracingSamplerRemoteURL is a constant for the GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL
+	// environment variable used to specify the remote url for the sampler type. This is relevant
+	// only when GF_INSTANCE_OTLP_SAMPLER_TYPE is "remote".
+	PluginTracingSamplerRemoteURL = "GF_INSTANCE_OTLP_SAMPLER_REMOTE_URL"
+
 	// PluginVersionEnv is a constant for the GF_PLUGIN_VERSION environment variable containing the plugin's version.
 	// Deprecated: Use build.GetBuildInfo().Version instead.
 	PluginVersionEnv = "GF_PLUGIN_VERSION"
+
+	// defaultRemoteSamplerServiceName is the default service name passed to the remote sampler when it cannot be
+	// determined from the build info.
+	defaultRemoteSamplerServiceName = "grafana-plugin"
 )
 
 // SetupPluginEnvironment will read the environment variables and apply the
@@ -117,17 +137,17 @@ func getTracerCustomAttributes(pluginID string) []attribute.KeyValue {
 // SetupTracer sets up the global OTEL trace provider and tracer.
 func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
 	// Set up tracing
-	tracingCfg := getTracingConfig()
-	if tracingCfg.IsEnabled() {
+	tracingCfg := getTracingConfig(build.GetBuildInfo)
+	if tracingCfg.isEnabled() {
 		// Append custom attributes to the default ones
 		tracingOpts.CustomAttributes = append(getTracerCustomAttributes(pluginID), tracingOpts.CustomAttributes...)
 
 		// Initialize global tracer provider
-		tp, err := tracerprovider.NewTracerProvider(tracingCfg.Address, tracingOpts)
+		tp, err := tracerprovider.NewTracerProvider(tracingCfg.address, tracingCfg.sampler, tracingOpts)
 		if err != nil {
 			return fmt.Errorf("new trace provider: %w", err)
 		}
-		pf, err := tracerprovider.NewTextMapPropagator(tracingCfg.Propagation)
+		pf, err := tracerprovider.NewTextMapPropagator(tracingCfg.propagation)
 		if err != nil {
 			return fmt.Errorf("new propagator format: %w", err)
 		}
@@ -136,30 +156,90 @@ func SetupTracer(pluginID string, tracingOpts tracing.Opts) error {
 		// Initialize global tracer for plugin developer usage
 		tracing.InitDefaultTracer(otel.Tracer(pluginID))
 	}
-	Logger.Debug("Tracing", "enabled", tracingCfg.IsEnabled(), "propagation", tracingCfg.Propagation)
+
+	enabled := tracingCfg.isEnabled()
+	Logger.Debug("Tracing", "enabled", enabled)
+	if enabled {
+		Logger.Debug(
+			"Tracing configuration",
+			"propagation", tracingCfg.propagation,
+			"samplerType", tracingCfg.sampler.SamplerType,
+			"samplerParam", tracingCfg.sampler.Param,
+			"samplerRemoteURL", tracingCfg.sampler.Remote.URL,
+			"samplerRemoteServiceName", tracingCfg.sampler.Remote.ServiceName,
+		)
+	}
 	return nil
 }
 
 // tracingConfig contains the configuration for OTEL tracing.
 type tracingConfig struct {
-	Address     string
-	Propagation string
+	address     string
+	propagation string
+
+	sampler tracerprovider.SamplerOptions
 }
 
-// IsEnabled returns true if OTEL tracing is enabled.
-func (c tracingConfig) IsEnabled() bool {
-	return c.Address != ""
+// isEnabled returns true if OTEL tracing is enabled.
+func (c tracingConfig) isEnabled() bool {
+	return c.address != ""
 }
 
 // getTracingConfig returns a new tracingConfig based on the current environment variables.
-func getTracingConfig() tracingConfig {
-	var otelAddr, otelPropagation string
+func getTracingConfig(buildInfoGetter build.InfoGetter) tracingConfig {
+	var otelAddr, otelPropagation, samplerRemoteURL, samplerParamString string
+	var samplerType tracerprovider.SamplerType
+	var samplerParam float64
 	otelAddr, ok := os.LookupEnv(PluginTracingOpenTelemetryOTLPAddressEnv)
 	if ok {
+		// Additional OTEL config
 		otelPropagation = os.Getenv(PluginTracingOpenTelemetryOTLPPropagationEnv)
+
+		// Sampling config
+		samplerType = tracerprovider.SamplerType(os.Getenv(PluginTracingSamplerTypeEnv))
+		samplerRemoteURL = os.Getenv(PluginTracingSamplerRemoteURL)
+		samplerParamString = os.Getenv(PluginTracingSamplerParamEnv)
+		var err error
+		samplerParam, err = strconv.ParseFloat(samplerParamString, 64)
+		if err != nil {
+			// Default value if invalid float is provided is 1.0 (AlwaysSample)
+			log.DefaultLogger.Warn(
+				"Could not parse sampler param to float, defaulting to 1.0",
+				"samplerParam", samplerParamString, "error", err,
+			)
+			samplerParam = 1.0
+		}
+	}
+	var serviceName string
+	if samplerType == tracerprovider.SamplerTypeRemote {
+		serviceName = remoteSamplerServiceName(buildInfoGetter)
 	}
 	return tracingConfig{
-		Address:     otelAddr,
-		Propagation: otelPropagation,
+		address:     otelAddr,
+		propagation: otelPropagation,
+		sampler: tracerprovider.SamplerOptions{
+			SamplerType: samplerType,
+			Param:       samplerParam,
+			Remote: tracerprovider.RemoteSamplerOptions{
+				URL:         samplerRemoteURL,
+				ServiceName: serviceName,
+			},
+		},
+	}
+}
+
+// remoteSamplerServiceName returns the service name for the remote tracing sampler.
+// It attempts to get it from the provided buildinfo getter. If unsuccessful or empty,
+// defaultRemoteSamplerServiceName is returned instead.
+func remoteSamplerServiceName(buildInfoGetter build.InfoGetter) string {
+	// Use plugin id as service name, if possible. Otherwise, use a generic default value.
+	bi, err := buildInfoGetter.GetInfo()
+	if err != nil {
+		log.DefaultLogger.Warn("Could not get build info for remote sampler service name", "error", err)
+		return defaultRemoteSamplerServiceName
+	}
+	if bi.PluginID == "" {
+		return defaultRemoteSamplerServiceName
 	}
+	return bi.PluginID
 }

backend/setup_test.go

@@ -0,0 +1,125 @@
+package backend
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/build"
+	"github.com/grafana/grafana-plugin-sdk-go/internal/tracerprovider"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetTracingConfig(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+
+		env             map[string]string
+		buildInfoGetter build.InfoGetter
+
+		expEnabled bool
+		expCfg     tracingConfig
+	}{
+		{
+			name:       "disabled",
+			env:        nil,
+			expEnabled: false,
+			expCfg:     tracingConfig{},
+		},
+		{
+			name: "otel with default sampler",
+			env: map[string]string{
+				PluginTracingOpenTelemetryOTLPAddressEnv:     "127.0.0.1:10000",
+				PluginTracingOpenTelemetryOTLPPropagationEnv: "jaeger",
+			},
+			expEnabled: true,
+			expCfg: tracingConfig{
+				address:     "127.0.0.1:10000",
+				propagation: "jaeger",
+				sampler: tracerprovider.SamplerOptions{
+					SamplerType: "",
+					Param:       1.0, // always sample
+					Remote:      tracerprovider.RemoteSamplerOptions{},
+				},
+			},
+		},
+		{
+			name: "otel with sampler and sampler param",
+			env: map[string]string{
+				PluginTracingOpenTelemetryOTLPAddressEnv:     "127.0.0.1:10000",
+				PluginTracingOpenTelemetryOTLPPropagationEnv: "jaeger",
+				PluginTracingSamplerTypeEnv:                  "rateLimiting",
+				PluginTracingSamplerParamEnv:                 "0.5",
+			},
+			expEnabled: true,
+			expCfg: tracingConfig{
+				address:     "127.0.0.1:10000",
+				propagation: "jaeger",
+				sampler: tracerprovider.SamplerOptions{
+					SamplerType: "rateLimiting",
+					Param:       0.5,
+					Remote:      tracerprovider.RemoteSamplerOptions{},
+				},
+			},
+		},
+		{
+			name: "otel with remote sampler",
+			env: map[string]string{
+				PluginTracingOpenTelemetryOTLPAddressEnv:     "127.0.0.1:10000",
+				PluginTracingOpenTelemetryOTLPPropagationEnv: "jaeger",
+				PluginTracingSamplerTypeEnv:                  "remote",
+				PluginTracingSamplerParamEnv:                 "0.5",
+				PluginTracingSamplerRemoteURL:                "127.0.0.1:10001",
+			},
+			expEnabled: true,
+			expCfg: tracingConfig{
+				address:     "127.0.0.1:10000",
+				propagation: "jaeger",
+				sampler: tracerprovider.SamplerOptions{
+					SamplerType: "remote",
+					Param:       0.5,
+					Remote: tracerprovider.RemoteSamplerOptions{
+						URL:         "127.0.0.1:10001",
+						ServiceName: "grafana-plugin",
+					},
+				},
+			},
+		},
+		{
+			name: "otel with remote sampler and buildinfo service name",
+			env: map[string]string{
+				PluginTracingOpenTelemetryOTLPAddressEnv:     "127.0.0.1:10000",
+				PluginTracingOpenTelemetryOTLPPropagationEnv: "jaeger",
+				PluginTracingSamplerTypeEnv:                  "remote",
+				PluginTracingSamplerParamEnv:                 "0.5",
+				PluginTracingSamplerRemoteURL:                "127.0.0.1:10001",
+			},
+			buildInfoGetter: build.InfoGetterFunc(func() (build.Info, error) {
+				return build.Info{PluginID: "my-example-datasource"}, nil
+			}),
+			expEnabled: true,
+			expCfg: tracingConfig{
+				address:     "127.0.0.1:10000",
+				propagation: "jaeger",
+				sampler: tracerprovider.SamplerOptions{
+					SamplerType: "remote",
+					Param:       0.5,
+					Remote: tracerprovider.RemoteSamplerOptions{
+						URL:         "127.0.0.1:10001",
+						ServiceName: "my-example-datasource",
+					},
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			for e, v := range tc.env {
+				t.Setenv(e, v)
+			}
+			if tc.buildInfoGetter == nil {
+				tc.buildInfoGetter = build.GetBuildInfo
+			}
+			cfg := getTracingConfig(tc.buildInfoGetter)
+			require.Equal(t, tc.expEnabled, cfg.isEnabled())
+			require.Equal(t, tc.expCfg, cfg)
+		})
+	}
+}

build/info.go

@@ -111,13 +111,26 @@ func getBuildInfoFromEnvironment() Info {
 	return v
 }
 
-// GetBuildInfo returns the build information that was compiled into the binary using:
+// InfoGetter is an interface with a method for returning the build info.
+type InfoGetter interface {
+	// GetInfo returns the build info.
+	GetInfo() (Info, error)
+}
+
+// InfoGetterFunc can be used to adapt ordinary functions into types satisfying the InfoGetter interface .
+type InfoGetterFunc func() (Info, error)
+
+func (f InfoGetterFunc) GetInfo() (Info, error) {
+	return f()
+}
+
+// GetBuildInfo is the default InfoGetter that returns the build information that was compiled into the binary using:
 // -X `github.com/grafana/grafana-plugin-sdk-go/build.buildInfoJSON={...}`
-func GetBuildInfo() (Info, error) {
+var GetBuildInfo = InfoGetterFunc(func() (Info, error) {
 	v := Info{}
 	if buildInfoJSON == "" {
 		return v, fmt.Errorf("build info was now set when this was compiled")
 	}
 	err := json.Unmarshal([]byte(buildInfoJSON), &v)
 	return v, err
-}
+})