Merge pull request #825 from Altinity/customizations/24.3.18

Author: Enmk

.github/create_workflow_report.py

@@ -253,11 +253,17 @@ def get_commit_statuses(sha: str) -> pd.DataFrame:
         for item in all_data
     ]
 
-    return (
-        pd.DataFrame(parsed)
-        .sort_values(by=["job_status", "job_name"], ascending=[True, True])
-        .reset_index(drop=True)
-    )
+    # Create DataFrame
+    df = pd.DataFrame(parsed)
+
+    # Drop duplicates keeping the first occurrence (newest status for each context)
+    # GitHub returns statuses in reverse chronological order
+    df = df.drop_duplicates(subset=["job_name"], keep="first")
+
+    # Sort by status and job name
+    return df.sort_values(
+        by=["job_status", "job_name"], ascending=[True, True]
+    ).reset_index(drop=True)
 
 
 def get_pr_info_from_number(pr_number: str) -> dict:
@@ -291,28 +297,50 @@ def get_checks_fails(client: Client, job_url: str):
     Get tests that did not succeed for the given job URL.
     Exclude checks that have status 'error' as they are counted in get_checks_errors.
     """
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND test_status IN ('FAIL', 'ERROR')
-                AND check_status!='error'
-                ORDER BY check_name, test_name
-                """
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+            FROM (
+                SELECT
+                    argMax(check_status, check_start_time) as job_status,
+                    check_name as job_name,
+                    argMax(test_status, check_start_time) as status,
+                    test_name,
+                    report_url as results_link,
+                    task_url
+                FROM `gh-data`.checks
+                GROUP BY check_name, test_name, report_url, task_url
+            )
+            WHERE task_url LIKE '{job_url}%'
+            AND test_status IN ('FAIL', 'ERROR')
+            AND job_status!='error'
+            ORDER BY job_name, test_name
+            """
     return client.query_dataframe(query)
 
 
 def get_checks_known_fails(client: Client, job_url: str, known_fails: dict):
     """
     Get tests that are known to fail for the given job URL.
     """
-    assert len(known_fails) > 0, "cannot query the database with empty known fails"
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND test_status='BROKEN'
-                AND test_name IN ({','.join(f"'{test}'" for test in known_fails.keys())})
-                ORDER BY test_name, check_name
-                """
+    if len(known_fails) == 0:
+        return pd.DataFrame()
+
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+        FROM (
+            SELECT
+                argMax(check_status, check_start_time) as job_status,
+                check_name as job_name,
+                argMax(test_status, check_start_time) as status,
+                test_name,
+                report_url as results_link,
+                task_url
+            FROM `gh-data`.checks
+            GROUP BY check_name, test_name, report_url, task_url
+        )
+        WHERE task_url LIKE '{job_url}%'
+        AND test_status='BROKEN'
+        AND test_name IN ({','.join(f"'{test}'" for test in known_fails.keys())})
+        ORDER BY job_name, test_name
+        """
 
     df = client.query_dataframe(query)
 
@@ -333,12 +361,22 @@ def get_checks_errors(client: Client, job_url: str):
     """
     Get checks that have status 'error' for the given job URL.
     """
-    columns = "check_status as job_status, check_name as job_name, test_status, test_name, report_url as results_link"
-    query = f"""SELECT {columns} FROM `gh-data`.checks
-                WHERE task_url LIKE '{job_url}%'
-                AND check_status=='error'
-                ORDER BY check_name, test_name
-                """
+    query = f"""SELECT job_status, job_name, status as test_status, test_name, results_link
+            FROM (
+                SELECT
+                    argMax(check_status, check_start_time) as job_status,
+                    check_name as job_name,
+                    argMax(test_status, check_start_time) as status,
+                    test_name,
+                    report_url as results_link,
+                    task_url
+                FROM `gh-data`.checks
+                GROUP BY check_name, test_name, report_url, task_url
+            )
+            WHERE task_url LIKE '{job_url}%'
+            AND job_status=='error'
+            ORDER BY job_name, test_name
+            """
     return client.query_dataframe(query)
 
 

.github/workflows/release_branches.yml

@@ -398,51 +398,51 @@ jobs:
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
 ##############################################################################################
-  StressTestAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (asan)
-      runner_type: altinity-func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (tsan)
-      runner_type: altinity-func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (msan)
-      runner_type: altinity-func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestUBsan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (ubsan)
-      runner_type: altinity-func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    secrets: inherit
-    with:
-      test_name: Stress test (debug)
-      runner_type: altinity-func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
+  # StressTestAsan:
+  #   needs: [RunConfig, BuilderDebAsan]
+  #   if: ${{ !failure() && !cancelled() }}
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   secrets: inherit
+  #   with:
+  #     test_name: Stress test (asan)
+  #     runner_type: altinity-func-tester
+  #     data: ${{ needs.RunConfig.outputs.data }}
+  # StressTestTsan:
+  #   needs: [RunConfig, BuilderDebTsan]
+  #   if: ${{ !failure() && !cancelled() }}
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   secrets: inherit
+  #   with:
+  #     test_name: Stress test (tsan)
+  #     runner_type: altinity-func-tester
+  #     data: ${{ needs.RunConfig.outputs.data }}
+  # StressTestMsan:
+  #   needs: [RunConfig, BuilderDebMsan]
+  #   if: ${{ !failure() && !cancelled() }}
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   secrets: inherit
+  #   with:
+  #     test_name: Stress test (msan)
+  #     runner_type: altinity-func-tester
+  #     data: ${{ needs.RunConfig.outputs.data }}
+  # StressTestUBsan:
+  #   needs: [RunConfig, BuilderDebUBsan]
+  #   if: ${{ !failure() && !cancelled() }}
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   secrets: inherit
+  #   with:
+  #     test_name: Stress test (ubsan)
+  #     runner_type: altinity-func-tester
+  #     data: ${{ needs.RunConfig.outputs.data }}
+  # StressTestDebug:
+  #   needs: [RunConfig, BuilderDebDebug]
+  #   if: ${{ !failure() && !cancelled() }}
+  #   uses: ./.github/workflows/reusable_test.yml
+  #   secrets: inherit
+  #   with:
+  #     test_name: Stress test (debug)
+  #     runner_type: altinity-func-tester
+  #     data: ${{ needs.RunConfig.outputs.data }}
 #############################################################################################
 ############################# INTEGRATION TESTS #############################################
 #############################################################################################
@@ -487,7 +487,7 @@ jobs:
 #############################################################################################
   RegressionTestsRelease:
     needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_options.exclude_keywords, 'regression')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -498,7 +498,7 @@ jobs:
       timeout_minutes: 300
   RegressionTestsAarch64:
     needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'aarch64')}}
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_options.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_options.exclude_keywords, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
@@ -546,11 +546,11 @@ jobs:
       - FunctionalStatefulTestTsan
       - FunctionalStatefulTestMsan
       - FunctionalStatefulTestUBsan
-      - StressTestDebug
-      - StressTestAsan
-      - StressTestTsan
-      - StressTestMsan
-      - StressTestUBsan
+      # - StressTestDebug
+      # - StressTestAsan
+      # - StressTestTsan
+      # - StressTestMsan
+      # - StressTestUBsan
       - IntegrationTestsAsan
       - IntegrationTestsTsan
       - IntegrationTestsRelease

docs/en/engines/table-engines/integrations/jdbc.md

@@ -39,6 +39,8 @@ ENGINE = JDBC(datasource_uri, external_database, external_table)
 
 - `external_table` — Name of the table in `external_database` or a select query like `select * from table1 where column1=1`.
 
+- These parameters can also be passed using [named collections](operations/named-collections.md).
+
 ## Usage Example {#usage-example}
 
 Creating a table in MySQL server by connecting directly with it’s console client:

docs/en/engines/table-engines/integrations/odbc.md

@@ -38,6 +38,8 @@ The table structure can differ from the source table structure:
 - `external_database` — Name of a database in an external DBMS.
 - `external_table` — Name of a table in the `external_database`.
 
+These parameters can also be passed using [named collections](operations/named-collections.md).
+
 ## Usage Example {#usage-example}
 
 **Retrieving data from the local MySQL installation via ODBC**

docs/en/sql-reference/table-functions/jdbc.md

@@ -11,11 +11,19 @@ clickhouse-jdbc-bridge contains experimental codes and is no longer supported. I
 ClickHouse recommend using built-in table functions in ClickHouse which provide a better alternative for ad-hoc querying scenarios (Postgres, MySQL, MongoDB, etc).
 :::
 
-`jdbc(datasource, schema, table)` - returns table that is connected via JDBC driver.
+JDBC table function returns table that is connected via JDBC driver.
 
 This table function requires separate [clickhouse-jdbc-bridge](https://github.com/ClickHouse/clickhouse-jdbc-bridge) program to be running.
 It supports Nullable types (based on DDL of remote table that is queried).
 
+## Syntax {#syntax}
+
+```sql
+jdbc(datasource, schema, table)
+jdbc(datasource, table)
+jdbc(named_collection)
+```
+
 **Examples**
 
 ``` sql

docs/en/sql-reference/table-functions/odbc.md

@@ -10,6 +10,8 @@ Returns table that is connected via [ODBC](https://en.wikipedia.org/wiki/Open_Da
 
 ``` sql
 odbc(connection_settings, external_database, external_table)
+odbc(connection_settings, external_table)
+odbc(named_collection)
 ```
 
 Parameters:
@@ -18,6 +20,8 @@ Parameters:
 - `external_database` — Name of a database in an external DBMS.
 - `external_table` — Name of a table in the `external_database`.
 
+These parameters can also be passed using [named collections](operations/named-collections.md).
+
 To safely implement ODBC connections, ClickHouse uses a separate program `clickhouse-odbc-bridge`. If the ODBC driver is loaded directly from `clickhouse-server`, driver problems can crash the ClickHouse server. ClickHouse automatically starts `clickhouse-odbc-bridge` when it is required. The ODBC bridge program is installed from the same package as the `clickhouse-server`.
 
 The fields with the `NULL` values from the external table are converted into the default values for the base data type. For example, if a remote MySQL table field has the `INT NULL` type it is converted to 0 (the default value for ClickHouse `Int32` data type).

src/Interpreters/AggregationCommon.h

@@ -132,6 +132,9 @@ static inline T ALWAYS_INLINE packFixed(
     {
         size_t index = i;
         const IColumn * column = key_columns[j];
+        auto column_possibly_converted = column->convertToFullColumnIfSparse()->convertToFullColumnIfConst();
+        column = column_possibly_converted.get();
+
         if constexpr (has_low_cardinality)
         {
             if (const IColumn * positions = (*low_cardinality_positions)[j])
@@ -224,26 +227,30 @@ static inline T ALWAYS_INLINE packFixed(
         if (is_null)
             continue;
 
+        const IColumn * key_column = key_columns[j];
+        auto column_possibly_converted = key_column->convertToFullColumnIfSparse()->convertToFullColumnIfConst();
+        key_column = column_possibly_converted.get();
+
         switch (key_sizes[j])
         {
             case 1:
-                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_columns[j])->getRawDataBegin<1>() + i, 1);
+                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_column)->getRawDataBegin<1>() + i, 1);
                 offset += 1;
                 break;
             case 2:
-                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_columns[j])->getRawDataBegin<2>() + i * 2, 2);
+                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_column)->getRawDataBegin<2>() + i * 2, 2);
                 offset += 2;
                 break;
             case 4:
-                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_columns[j])->getRawDataBegin<4>() + i * 4, 4);
+                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_column)->getRawDataBegin<4>() + i * 4, 4);
                 offset += 4;
                 break;
             case 8:
-                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_columns[j])->getRawDataBegin<8>() + i * 8, 8);
+                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_column)->getRawDataBegin<8>() + i * 8, 8);
                 offset += 8;
                 break;
             default:
-                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_columns[j])->getRawDataBegin<1>() + i * key_sizes[j], key_sizes[j]);
+                memcpy(bytes + offset, static_cast<const ColumnFixedSizeHelper *>(key_column)->getRawDataBegin<1>() + i * key_sizes[j], key_sizes[j]);
                 offset += key_sizes[j];
         }
     }