SLO - http request external duration and error source (#989)

* SLO - http request external duration and error source
---------

Co-authored-by: Giuseppe Guerra <[email protected]>

Author: scottlepp

experimental/slo/slo_middleware.go

@@ -0,0 +1,102 @@
+package slo
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var Logger = log.DefaultLogger
+
+var duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Namespace: "plugins",
+	Name:      "plugin_external_requests_duration_seconds",
+	Help:      "Duration of requests to external services",
+}, []string{"datasource_name", "datasource_type", "error_source"})
+
+const DataSourceSLOMiddlewareName = "slo"
+
+// Middleware captures duration of requests to external services and the source of errors
+func Middleware() httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(DataSourceSLOMiddlewareName, RoundTripper)
+}
+
+// RoundTripper captures duration of requests to external services and the source of errors
+func RoundTripper(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+	name, kind, err := getDSInfo(opts)
+	if err != nil {
+		Logger.Error("failed to get datasource info", "error", err)
+		return next
+	}
+	return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		start := time.Now()
+		var errorSource = "none"
+
+		defer func() {
+			duration.WithLabelValues(name, kind, errorSource).Observe(time.Since(start).Seconds())
+		}()
+
+		res, err := next.RoundTrip(req)
+		if res != nil && res.StatusCode >= 400 {
+			errorSource = string(backend.ErrorSourceFromHTTPStatus(res.StatusCode))
+		}
+		if errors.Is(err, syscall.ECONNREFUSED) {
+			errorSource = string(backend.ErrorSourceDownstream)
+		}
+		return res, err
+	})
+}
+
+func getDSInfo(opts httpclient.Options) (string, string, error) {
+	datasourceName, exists := opts.Labels["datasource_name"]
+	if !exists {
+		return "", "", errors.New("datasource_name label not found")
+	}
+
+	datasourceName, err := SanitizeLabelName(datasourceName)
+	// if the datasource named cannot be turned into a prometheus
+	// label we will skip instrumenting these metrics.
+	if err != nil {
+		return "", "", err
+	}
+
+	datasourceType, exists := opts.Labels["datasource_type"]
+	if !exists {
+		return "", "", errors.New("datasource_type label not found")
+	}
+
+	return datasourceName, datasourceType, nil
+}
+
+// SanitizeLabelName removes all invalid chars from the label name.
+// If the label name is empty or contains only invalid chars, it
+// will return an error.
+func SanitizeLabelName(name string) (string, error) {
+	if len(name) == 0 {
+		return "", errors.New("label name cannot be empty")
+	}
+
+	out := strings.Builder{}
+	for i, b := range name {
+		if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0) {
+			out.WriteRune(b)
+		} else if b == ' ' {
+			out.WriteRune('_')
+		}
+	}
+
+	if out.Len() == 0 {
+		return "", fmt.Errorf("label name only contains invalid chars: %q", name)
+	}
+
+	return out.String(), nil
+}

experimental/testdata/folder.golden.txt

@@ -9,7 +9,7 @@ Frame[0] {
     "pathSeparator": "/"
 }
 Name: 
-Dimensions: 2 Fields by 21 Rows
+Dimensions: 2 Fields by 22 Rows
 +----------------+------------------+
 | Name: name     | Name: media-type |
 | Labels:        | Labels:          |
@@ -29,4 +29,4 @@ Dimensions: 2 Fields by 21 Rows
 
 
 ====== TEST DATA RESPONSE (arrow base64) ======
-FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAMAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABUAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABYAAAAAAAAAFgAAAAAAAAAAQEAAAAAAABgAQAAAAAAAAAAAAAAAAAAYAEAAAAAAABYAAAAAAAAALgBAAAAAAAAdQAAAAAAAAAAAAAAAgAAABUAAAAAAAAAAAAAAAAAAAAVAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAFAAAAB4AAAAoAAAANgAAADkAAABEAAAAUgAAAF0AAABtAAAAfAAAAJAAAACqAAAAyQAAANQAAADaAAAA3gAAAOwAAAD5AAAAAQEAAFJFQURNRS5tZGFjdGlvbnNhcGlzYXV0aGNsaWVudGNvbmN1cnJlbnRkYXRhc291cmNldGVzdGUyZWVycm9yc291cmNlZmVhdHVyZXRvZ2dsZXNmaWxlaW5mby5nb2ZpbGVpbmZvX3Rlc3QuZ29mcmFtZV9zb3J0ZXIuZ29mcmFtZV9zb3J0ZXJfdGVzdC5nb2dvbGRlbl9yZXNwb25zZV9jaGVja2VyLmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXJfdGVzdC5nb2h0dHBfbG9nZ2VybWFjcm9zbW9ja3Jlc3RfY2xpZW50Lmdvc2NoZW1hYnVpbGRlcnRlc3RkYXRhAAAAAAAAAAAAAAAAAAAACQAAABIAAAAbAAAAJAAAAC0AAAA2AAAAPwAAAEgAAABIAAAASAAAAEgAAABIAAAASAAAAEgAAABRAAAAWgAAAGMAAABjAAAAbAAAAHUAAABkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnlkaXJlY3RvcnkAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAA2AEAAAAAAADgAAAAAAAAADACAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAEwAAAAoAAAABAAAAMD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA4P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAA////CAAAAFAAAABEAAAAeyJ0eXBlIjoiZGlyZWN0b3J5LWxpc3RpbmciLCJ0eXBlVmVyc2lvbiI6WzAsMF0sInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHgAAAAEAAAAov///xQAAAA8AAAAPAAAAAAAAAU4AAAAAQAAAAQAAACQ////CAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAAD4AQAAQVJST1cx
+FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAALgAAAADAAAATAAAACgAAAAEAAAAwP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADg/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAD///8IAAAAUAAAAEQAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInR5cGVWZXJzaW9uIjpbMCwwXSwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAeAAAAAQAAACi////FAAAADwAAAA8AAAAAAAABTgAAAABAAAABAAAAJD///8IAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAACI////CgAAAG1lZGlhLXR5cGUAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAYAAABzdHJpbmcAAAYAAAB0c3R5cGUAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP/////YAAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAASAIAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAeAAAABYAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABcAAAAAAAAAGAAAAAAAAAABAEAAAAAAABoAQAAAAAAAAAAAAAAAAAAaAEAAAAAAABcAAAAAAAAAMgBAAAAAAAAfgAAAAAAAAAAAAAAAgAAABYAAAAAAAAAAAAAAAAAAAAWAAAAAAAAAAAAAAAAAAAAAAAAAAkAAAAQAAAAFAAAAB4AAAAoAAAANgAAADkAAABEAAAAUgAAAF0AAABtAAAAfAAAAJAAAACqAAAAyQAAANQAAADaAAAA3gAAAOwAAAD5AAAA/AAAAAQBAAAAAAAAUkVBRE1FLm1kYWN0aW9uc2FwaXNhdXRoY2xpZW50Y29uY3VycmVudGRhdGFzb3VyY2V0ZXN0ZTJlZXJyb3Jzb3VyY2VmZWF0dXJldG9nZ2xlc2ZpbGVpbmZvLmdvZmlsZWluZm9fdGVzdC5nb2ZyYW1lX3NvcnRlci5nb2ZyYW1lX3NvcnRlcl90ZXN0LmdvZ29sZGVuX3Jlc3BvbnNlX2NoZWNrZXIuZ29nb2xkZW5fcmVzcG9uc2VfY2hlY2tlcl90ZXN0LmdvaHR0cF9sb2dnZXJtYWNyb3Ntb2NrcmVzdF9jbGllbnQuZ29zY2hlbWFidWlsZGVyc2xvdGVzdGRhdGEAAAAAAAAAAAAAAAAJAAAAEgAAABsAAAAkAAAALQAAADYAAAA/AAAASAAAAEgAAABIAAAASAAAAEgAAABIAAAASAAAAFEAAABaAAAAYwAAAGMAAABsAAAAdQAAAH4AAAAAAAAAZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5ZGlyZWN0b3J5AAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAABAABAAAA2AEAAAAAAADgAAAAAAAAAEgCAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAAC4AAAAAwAAAEwAAAAoAAAABAAAAMD+//8IAAAADAAAAAAAAAAAAAAABQAAAHJlZklkAAAA4P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAAA////CAAAAFAAAABEAAAAeyJ0eXBlIjoiZGlyZWN0b3J5LWxpc3RpbmciLCJ0eXBlVmVyc2lvbiI6WzAsMF0sInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHgAAAAEAAAAov///xQAAAA8AAAAPAAAAAAAAAU4AAAAAQAAAAQAAACQ////CAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAGAAAAc3RyaW5nAAAGAAAAdHN0eXBlAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAAD4AQAAQVJST1cx