No e2 e stats (#2374)

* exclude E2E-TEST realm from system code stats

* better error

* moar compact sntaxes

* pivot to list of IDs

* lint

Author: mikehelmick

pkg/config/server_config.go

@@ -103,6 +103,9 @@ type ServerConfig struct {
 	// MinRealmsForSystemStatistics gives a minimum threshold for displaying system
 	// admin level statistics
 	MinRealmsForSystemStatistics uint `env:"MIN_REALMS_FOR_SYSTEM_STATS, default=2"`
+	// ExcludeFromSystemStatistics is a list of realm IDs to exclude from system stats, in addition
+	// to any e2e test realms that are found.
+	ExcludeFromSystemStatistics []uint `env:"EXCLUDE_FROM_SYSTEM_STATS"`
 
 	// Rate limiting configuration
 	RateLimit ratelimit.Config

pkg/controller/admin/code_stats.go

@@ -26,7 +26,7 @@ func (c *Controller) HandleCodeStats() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 
-		stats, err := c.db.AllRealmCodeStatsCached(ctx, c.cacher, int(c.config.MinRealmsForSystemStatistics))
+		stats, err := c.db.AllRealmCodeStatsCached(ctx, c.cacher, int(c.config.MinRealmsForSystemStatistics), c.config.ExcludeFromSystemStatistics)
 		if err != nil {
 			controller.InternalError(w, r, c.h, err)
 			return

pkg/database/realm.go

@@ -1323,6 +1323,18 @@ func (db *Database) MaximumUserReportTimeout() (time.Duration, error) {
 	return timeout, nil
 }
 
+func (db *Database) FindE2ETestRealm() (*Realm, error) {
+	var realm Realm
+	if err := db.db.
+		Model(&Realm{}).
+		Where("is_e2e = ?", true).
+		First(&realm).
+		Error; err != nil {
+		return nil, err
+	}
+	return &realm, nil
+}
+
 func (db *Database) FindRealmByRegion(region string) (*Realm, error) {
 	var realm Realm
 	if err := db.db.
@@ -1924,7 +1936,7 @@ func (r *Realm) Stats(db *Database) (RealmStats, error) {
 
 // AllRealmCodeStatsCached returns combined code issue / claimed stats for all realms
 // in the system, but cached.
-func (db *Database) AllRealmCodeStatsCached(ctx context.Context, cacher cache.Cacher, requiredRealms int) (RealmStats, error) {
+func (db *Database) AllRealmCodeStatsCached(ctx context.Context, cacher cache.Cacher, requiredRealms int, excludeRealmIDs []uint) (RealmStats, error) {
 	if cacher == nil {
 		return nil, fmt.Errorf("cacher cannot be nil")
 	}
@@ -1935,7 +1947,7 @@ func (db *Database) AllRealmCodeStatsCached(ctx context.Context, cacher cache.Ca
 		Key:       fmt.Sprintf("all:min-%d", requiredRealms),
 	}
 	if err := cacher.Fetch(ctx, cacheKey, &stats, 30*time.Minute, func() (interface{}, error) {
-		return db.AllRealmCodeStats(ctx, requiredRealms)
+		return db.AllRealmCodeStats(ctx, requiredRealms, excludeRealmIDs)
 	}); err != nil {
 		return nil, err
 	}
@@ -1947,13 +1959,24 @@ func (db *Database) AllRealmCodeStatsCached(ctx context.Context, cacher cache.Ca
 // This reuses the RealmStats data structure and the realm_id is the COUNT
 // of all realms that contributed to statistics on that day, so that filtering
 // can be done if there are not enough data points.
-func (db *Database) AllRealmCodeStats(ctx context.Context, requiredRealms int) (RealmStats, error) {
+func (db *Database) AllRealmCodeStats(ctx context.Context, requiredRealms int, excludeRealmIDs []uint) (RealmStats, error) {
 	stop := timeutils.UTCMidnight(time.Now())
 	start := stop.Add(project.StatsDisplayDays * -24 * time.Hour)
 	if start.After(stop) {
 		return nil, ErrBadDateRange
 	}
 
+	allExcludedRealmIDs := make([]uint, 0)
+	// Find and exclude e2e test realm if it exists.
+	e2eTestRealm, err := db.FindE2ETestRealm()
+	if err != nil {
+		db.logger.Warnf("unable o find e2e test realms for exclusion from system stats", "error", err)
+	} else {
+		allExcludedRealmIDs = append(allExcludedRealmIDs, e2eTestRealm.ID)
+	}
+	// Exclude other explicitly listed realms
+	allExcludedRealmIDs = append(allExcludedRealmIDs, excludeRealmIDs...)
+
 	sql := `
 		SELECT
 			d.date AS date,
@@ -1974,11 +1997,14 @@ func (db *Database) AllRealmCodeStats(ctx context.Context, requiredRealms int) (
 			SELECT date::date FROM generate_series($1, $2, '1 day'::interval) date
 		) d
 		LEFT JOIN realm_stats s ON s.date = d.date
+		WHERE NOT s.realm_id = ANY ($3)
 		GROUP BY d.date
 		ORDER BY date DESC`
 
+	values := []any{start, stop, pq.Array(allExcludedRealmIDs)}
+
 	var stats []*RealmStat
-	if err := db.db.Raw(sql, start, stop).Scan(&stats).Error; err != nil {
+	if err := db.db.Raw(sql, values).Scan(&stats).Error; err != nil {
 		if IsNotFound(err) {
 			return stats, nil
 		}