Improve the time series api + add policy for regression (#7156)

For API:
- Add model filters during query
- Add format options table, and raw
- add API hook method for frontend

add listCommits api

For regression lambda
- Add regression policy for compilation latency (if new value> 1.05 x
baseline, consider as regression)
- Change the data format to match with the api

Author: yangw-dev

aws/lambda/benchmark_regression_summary_report/common/benchmark_time_series_api_model.py

@@ -58,7 +58,7 @@ def from_request(
             tr = TimeRange(**payload["data"]["time_range"])
             ts = [
                 BenchmarkTimeSeriesItem(**item)
-                for item in payload["data"]["time_series"]
+                for item in payload["data"]["data"]["time_series"]
             ]
         except Exception as e:
             raise RuntimeError(f"Malformed API payload: {e}")

aws/lambda/benchmark_regression_summary_report/common/config.py

@@ -27,6 +27,7 @@
         api_endpoint_params_template="""
                 {
                   "name": "compiler_precompute",
+                  "response_formats":["time_series"],
                   "query_params": {
                     "commits": [],
                     "compilers": [],
@@ -50,7 +51,7 @@
     policy=Policy(
         frequency=Frequency(value=1, unit="days"),
         range=RangeConfig(
-            baseline=DayRangeWindow(value=7),
+            baseline=DayRangeWindow(value=5),
             comparison=DayRangeWindow(value=2),
         ),
         metrics={
@@ -72,6 +73,12 @@
                 threshold=0.95,
                 baseline_aggregation="max",
             ),
+            "compilation_latency": RegressionPolicy(
+                name="compilation_latency",
+                condition="less_equal",
+                threshold=1.05,
+                baseline_aggregation="min",
+            ),
         },
         notification_config={
             "type": "github",

torchci/clickhouse_queries/compilers_benchmark_api_commit_query/query.sql

@@ -1,27 +1,45 @@
 SELECT DISTINCT
     replaceOne(head_branch, 'refs/heads/', '') AS branch,
     head_sha AS commit,
-    workflow_id AS id,
-    timestamp
+    workflow_id,
+    toDate(fromUnixTimestamp(timestamp), 'UTC') AS date
 FROM benchmark.oss_ci_benchmark_torchinductor
 PREWHERE
     timestamp >= toUnixTimestamp({startTime: DateTime64(3)})
     AND timestamp < toUnixTimestamp({stopTime:  DateTime64(3)})
 WHERE
+    -- optional branches
     (
         has(
             {branches: Array(String)},
             replaceOne(head_branch, 'refs/heads/', '')
         )
         OR empty({branches: Array(String)})
     )
+    -- optional suites
     AND (
-        has({suites: Array(String) }, suite)
-        OR empty({suites: Array(String) })
+        has({suites: Array(String)}, suite)
+        OR empty({suites: Array(String)})
     )
-    AND benchmark_dtype = {dtype: String}
-    AND benchmark_mode = {mode: String}
-    AND device = {device: String}
-    AND multiSearchAnyCaseInsensitive(arch, {arch: Array(String)})
-ORDER BY timestamp
+    -- optional dtype
+    AND (
+        benchmark_dtype = {dtype: String}
+        OR empty({dtype: String})
+    )
+    -- optional mode
+    AND (
+        benchmark_mode = {mode: String}
+        OR empty({mode: String})
+    )
+    -- optional device
+    AND (
+        device = {device: String}
+        OR empty({device: String})
+    )
+    -- optional arch (array param); if empty array, skip filter
+    AND (
+        multiSearchAnyCaseInsensitive(arch, {arch: Array(String)})
+        OR empty({arch: Array(String)})
+    )
+ORDER BY branch, timestamp
 SETTINGS session_timezone = 'UTC';

torchci/clickhouse_queries/compilers_benchmark_api_query/params.json

@@ -8,7 +8,8 @@
     "dtype": "String",
     "granularity": "String",
     "mode": "String",
-    "suites": "Array(String)"
+    "suites": "Array(String)",
+    "models": "Array(String)"
   },
   "tests": []
 }

torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql

@@ -30,6 +30,10 @@ WHERE
         has({suites: Array(String) }, suite)
         OR empty({suites: Array(String) })
     )
+    AND (
+        has({models: Array(String)}, model_name)
+        OR empty({models: Array(String) })
+    )
     AND benchmark_dtype = {dtype: String}
     AND benchmark_mode = {mode: String}
     AND device = {device: String}

torchci/components/benchmark/compilers/SummaryPanel.tsx

@@ -470,7 +470,6 @@ export function SummaryPanel({
                       if (l === r) {
                         return "";
                       }
-
                       // Decreasing more than x%
                       if (r - l > RELATIVE_THRESHOLD * r) {
                         return styles.ok;

torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts

@@ -7,7 +7,11 @@ import {
 } from "./helpers/common";
 import { toGeneralCompilerData } from "./helpers/general";
 import { toPrecomputeCompilerData } from "./helpers/precompute";
-import { CompilerQueryType } from "./type";
+import {
+  CompilerQueryType,
+  defaultGetTimeSeriesInputs,
+  defaultListCommitsInputs,
+} from "./type";
 //["x86_64","NVIDIA A10G","NVIDIA H100 80GB HBM3"]
 const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query";
 const COMPILER_BENCHMARK_COMMITS_TABLE_NAME =
@@ -16,7 +20,7 @@ const COMPILER_BENCHMARK_COMMITS_TABLE_NAME =
 export async function getCompilerBenchmarkData(
   inputparams: any,
   type: CompilerQueryType = CompilerQueryType.PRECOMPUTE,
-  format: string = "time_series"
+  formats: string[] = ["time_series"]
 ) {
   const rows = await getCompilerDataFromClickhouse(inputparams);
 
@@ -26,55 +30,85 @@ export async function getCompilerBenchmarkData(
 
   switch (type) {
     case CompilerQueryType.PRECOMPUTE:
-      return toPrecomputeCompilerData(rows, format);
+      return toPrecomputeCompilerData(rows, formats);
     case CompilerQueryType.GENERAL:
-      return toGeneralCompilerData(rows, format);
+      return toGeneralCompilerData(rows, formats);
     default:
       throw new Error(`Invalid compiler query type, got ${type}`);
   }
 }
 
+export async function getCompilerCommits(inputparams: any): Promise<any[]> {
+  if (!inputparams.startTime || !inputparams.stopTime) {
+    throw new Error("no start/end time provided in request");
+  }
+  const queryParams = {
+    ...defaultListCommitsInputs, // base defaults
+    ...inputparams, // override with caller's values
+  };
+
+  if (queryParams.arch && queryParams.device) {
+    const arch_list = toQueryArch(inputparams.device, inputparams.arch);
+    queryParams["arch"] = arch_list;
+  }
+
+  const commit_results = await queryClickhouseSaved(
+    COMPILER_BENCHMARK_COMMITS_TABLE_NAME,
+    queryParams
+  );
+  return commit_results;
+}
+
 async function getCompilerDataFromClickhouse(inputparams: any): Promise<any[]> {
   const start = Date.now();
-  const arch_list = toQueryArch(inputparams.device, inputparams.arch);
-  inputparams["arch"] = arch_list;
+
+  const queryParams = {
+    ...defaultGetTimeSeriesInputs, // base defaults
+    ...inputparams, // override with caller's values
+  };
+
+  if (queryParams.arch && queryParams.device) {
+    const arch_list = toQueryArch(queryParams.device, queryParams.arch);
+    queryParams["arch"] = arch_list;
+  }
 
   // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided
-  if (!inputparams.commits || inputparams.commits.length == 0) {
-    if (!inputparams.startTime || !inputparams.stopTime) {
+  if (!queryParams.commits || queryParams.commits.length == 0) {
+    if (!queryParams.startTime || !queryParams.stopTime) {
       console.log("no commits or start/end time provided in request");
       return [];
     }
+
     // get commits from clickhouse
     const commit_results = await queryClickhouseSaved(
       COMPILER_BENCHMARK_COMMITS_TABLE_NAME,
-      inputparams
+      queryParams
     );
     // get unique commits
     const unique_commits = [...new Set(commit_results.map((c) => c.commit))];
     if (unique_commits.length === 0) {
-      console.log("no commits found in clickhouse using", inputparams);
+      console.log("no commits found in clickhouse using", queryParams);
       return [];
     }
 
     console.log(
-      "no commits provided in request, found unqiue commits",
-      unique_commits
+      `no commits provided in request, searched unqiue commits based on
+      start/end time unique_commits: ${unique_commits.length}`
     );
 
     if (commit_results.length > 0) {
-      inputparams["commits"] = unique_commits;
+      queryParams["commits"] = unique_commits;
     } else {
-      console.log(`no commits found in clickhouse using ${inputparams}`);
+      console.log(`no commits found in clickhouse using ${queryParams}`);
       return [];
     }
   } else {
-    console.log("commits provided in request", inputparams.commits);
+    console.log("commits provided in request", queryParams.commits);
   }
 
   let rows = await queryClickhouseSaved(
     COMPILER_BENCHMARK_TABLE_NAME,
-    inputparams
+    queryParams
   );
   const end = Date.now();
   console.log("time to get compiler timeseris data", end - start);
@@ -83,6 +117,8 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise<any[]> {
     return [];
   }
 
+  console.log("rows from clickhouse", rows[0]);
+
   // extract backend from output in runtime instead of doing it in the query. since it's expensive for regex matching.
   // TODO(elainewy): we should add this as a column in the database for less runtime logics.
   rows.map((row) => {

torchci/lib/benchmark/api_helper/compilers/helpers/common.ts

@@ -1,14 +1,3 @@
-import { groupByBenchmarkData } from "../../utils";
-
-export function to_table_compiler_data(data: any[]) {
-  const res = groupByBenchmarkData(
-    data,
-    ["dtype", "arch", "device", "mode", "workflow_id", "granularity_bucket"],
-    ["metric", "compiler"]
-  );
-  return res;
-}
-
 export function extractBackendSqlStyle(
   output: string,
   suite: string,

torchci/lib/benchmark/api_helper/compilers/helpers/general.ts

@@ -13,6 +13,7 @@ const COMPILER_GENERAL_TS_GROUP_KEY = [
   "metric",
   "mode",
   "model",
+  "branch",
 ];
 const COMPILER_GENERAL_TS_SUB_GROUP_KEY = ["workflow_id"];
 
@@ -22,6 +23,7 @@ const COMPILER_GENERAL_TABLE_GROUP_KEY = [
   "device",
   "mode",
   "workflow_id",
+  "branch",
   "compiler",
   "model",
 ];
@@ -35,31 +37,40 @@ const COMPILER_GENERAL_TABLE_SUB_GROUP_KEY = ["metric"];
  */
 export function toGeneralCompilerData(
   rawData: any[],
-  type: string = "time_series"
+  formats: string[] = ["time_series"]
 ) {
   const start_ts = new Date(rawData[0].granularity_bucket).getTime();
   const end_ts = new Date(
     rawData[rawData.length - 1].granularity_bucket
   ).getTime();
 
-  let res: any[] = [];
-  switch (type) {
+  let formats_result: any = {};
+
+  formats.forEach((format) => {
+    const data = getformat(rawData, format);
+    formats_result[format] = data;
+  });
+  return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts);
+}
+
+function getformat(data: any, format: string) {
+  switch (format) {
     case "time_series":
-      res = to_time_series_data(
-        rawData,
+      return to_time_series_data(
+        data,
         COMPILER_GENERAL_TS_GROUP_KEY,
         COMPILER_GENERAL_TS_SUB_GROUP_KEY
       );
-      break;
     case "table":
-      res = groupByBenchmarkData(
-        rawData,
+      return groupByBenchmarkData(
+        data,
         COMPILER_GENERAL_TABLE_GROUP_KEY,
         COMPILER_GENERAL_TABLE_SUB_GROUP_KEY
       );
       break;
+    case "raw":
+      return data;
     default:
       throw new Error("Invalid type");
   }
-  return toTimeSeriesResponse(res, rawData.length, start_ts, end_ts);
 }