use regex to match sanitizer timeouts

Author: strtgbb

ci/jobs/scripts/integration_tests_runner.py

@@ -579,7 +579,7 @@ def matches_substring(substring, log, is_regex):
                     )
 
                     # Should only care about the most recent log file
-                    log_path = get_log_paths(log_paths, failed_test)[0]
+                    log_path = get_log_paths(failed_test)[0]
                     test_logs = extract_fail_logs(log_path)
                     test_log = test_logs.get(failed_test.split("::")[-1])
                     if test_log is None:

tests/broken_tests.yaml

@@ -1,17 +1,9 @@
+# Name fields should be unique.
 - name: 02844_max_backup_bandwidth_s3
   reason: INVESTIGATE - Fails on debug
   message: result differs with reference
   check_types:
   - debug
-- name: 03455_direct_io_read_array_values
-  reason: INVESTIGATE - Unstable on tsan
-  check_types:
-  - tsan
-- name: 00974_text_log_table_not_empty
-  reason: INVESTIGATE - Unstable on tsan
-  message: 'curl: (28) Operation timed out'
-  check_types:
-  - tsan
 - name: 00626_replace_partition_from_table
   reason: INVESTIGATE - Unstable on tsan
   message: 'DB::Exception: Cannot OPTIMIZE table: Current background tasks memory
@@ -31,201 +23,23 @@
 - name: 02443_detach_attach_partition
   reason: INVESTIGATE - Unstable
   message: 'Cannot rename parts to perform operation on them: Code: 521. DB::ErrnoException: Cannot rename'
-- name: 03568_mutation_affected_rows_counter
-  reason: INVESTIGATE - Unstable on tsan
-  message: 'DB::Exception: Timeout exceeded (180 s) while flushing system log'
-  check_types:
-  - tsan
-- name: 03100_lwu_03_join
-  reason: INVESTIGATE - Unstable on tsan
-  message: 'DB::Exception: Timeout exceeded (180 s) while flushing system log'
-  check_types:
-  - tsan
-- name: 02479_race_condition_between_insert_and_droppin_mv
-  reason: INVESTIGATE - Timeout in ClickHouse query
-  check_types:
-  - tsan
-  - msan
-- name: 01037_polygon_dicts_correctness_fast
-  reason: INVESTIGATE - Timeout in ClickHouse query
-  check_types:
-  - tsan
-  - msan
 - name: 03441_deltalake_clickhouse_public_datasets
   reason: INVESTIGATE - S3 sometimes unreachable
   message: 'Error interacting with object store: Generic S3 error: Error performing GET'
 - name: 03441_deltalake_clickhouse_virtual_columns
   reason: INVESTIGATE - S3 sometimes unreachable
   message: 'Error interacting with object store: Generic S3 error: Error performing GET'
-- name: 03357_join_pk_sharding
-  reason: INVESTIGATE - Timeout. Unstable?
-  check_types:
-  - tsan
-  - msan
-- name: 01630_simple_aggregate_all_functions_in_summing_merge_tree
-  reason: INVESTIGATE - Unstable?
-  check_types:
-  - tsan
-- name: 03514_grace_hash_join_logical_error
-  reason: INVESTIGATE - timeout
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
 - name: 00167_read_bytes_from_fs
   reason: INVESTIGATE - Memory limit exceeded
   message: 'DB::Exception: (total) memory limit exceeded:'
 - name: 02700_s3_part_INT_MAX
   reason: INVESTIGATE - Memory limit exceeded
   message: 'DB::Exception: (total) memory limit exceeded:'
-- name: 01903_correct_block_size_prediction_with_default
-  reason: INVESTIGATE - Unstable on tsan
-  check_types:
-  - tsan
-- name: 00157_cache_dictionary
-  reason: INVESTIGATE - Timeout/Memory limit exceeded in ClickHouse query
-- name: 01655_plan_optimizations_optimize_read_in_window_order_long
-  reason: INVESTIGATE - Timeout. Unstable on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02473_optimize_old_parts
-  reason: INVESTIGATE - Unstable on tsan
-  check_types:
-  - tsan
-- name: 03216_arrayWithConstant_limits
-  reason: INVESTIGATE - Timeout. Unstable on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02099_hashed_array_dictionary_complex_key
-  reason: INVESTIGATE - Timeout. Unstable on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 03305_mergine_aggregated_filter_push_down
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02760_dictionaries_memory
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 03532_json_dynamic_updates
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 03094_grouparraysorted_memory
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02784_parallel_replicas_automatic_decision
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 01184_long_insert_values_huge_strings
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02389_analyzer_nested_lambda
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout
-  check_types:
-  - tsan
-- name: 02581_share_big_sets_between_mutation_tasks
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 02581_share_big_sets_between_mutation_tasks_with_storage_set
-  reason: INVESTIGATE - Timeout on tsan
-  check_types:
-  - tsan
-- name: 00596_limit_on_expanded_ast
-  reason: INVESTIGATE - Timeout on tsan, msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-  - msan
-- name: 00816_long_concurrent_alter_column
-  reason: INVESTIGATE - Timeout on msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - msan
-- name: 00377_shard_group_uniq_array_of_string_array
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 01799_long_uniq_theta_sketch
-  reason: INVESTIGATE - Unstable on tsan
-  check_types:
-  - tsan
 - name: 03572_export_merge_tree_part_to_object_storage
-  reason: INVESTIGATE - Unstable on tsan
+  reason: INVESTIGATE - Unstable on asan, tsan
   message: result differs with reference
   check_types:
-  - tsan
-- name: 00172_hits_joins
-  reason: INVESTIGATE - Timeout on tsan, msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-  - msan
-- name: 00084_external_aggregation
-  reason: INVESTIGATE - Timeout on tsan, msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-  - msan
-- name: 03127_system_unload_primary_key_table
-  reason: INVESTIGATE - Timeout on msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - msan
-- name: 00184_parallel_hash_returns_same_res_as_hash
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-- name: 00429_long_http_bufferization
-  reason: INVESTIGATE - Timeout on tsan, msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - tsan
-  - msan
-- name: 00159_parallel_formatting_json_and_friends_1
-  reason: INVESTIGATE - Timeout on msan
-  message: Timeout! Processes left in process group
-  check_types:
-  - msan
-- name: 02177_issue_31009
-  reason: INVESTIGATE - Timeout on ubsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - ubsan
-- name: 00159_parallel_formatting_csv_and_friends
-  reason: INVESTIGATE - Timeout on ubsan
-  message: Timeout! Processes left in process group
-  check_types:
-  - ubsan
-- name: 00024_random_counters
-  reason: INVESTIGATE - Timeout on sanitizers
-  message: Timeout! Processes left in process group
-- name: 00159_parallel_formatting_json_and_friends_2
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout
-  check_types:
-  - tsan
-- name: 01782_field_oom
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout
-  check_types:
+  - asan
   - tsan
 - name: 03321_clickhouse_local_initialization_not_too_slow_even_under_sanitizers
   reason: INVESTIGATE - Unstable on msan
@@ -236,63 +50,6 @@
   message: 'DB::Exception: Estimated query execution time'
   check_types:
   - tsan
-- name: 03096_order_by_system_tables
-  reason: INVESTIGATE - Unstable on tsan
-  message: Timeout
-  check_types:
-  - tsan
-- name: 02210_processors_profile_log
-  reason: INVESTIGATE - Unstable on tsan
-  message: Timeout
-  check_types:
-  - tsan
-- name: 03555_json_shared_data_advanced_paths_indexes
-  reason: INVESTIGATE - Timeout on tsan, msan
-  message: Timeout
-  check_types:
-  - tsan
-  - msan
-- name: 02935_parallel_replicas_settings
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout exceeded (180 s) while flushing system log
-  check_types:
-  - tsan
-- name: 03100_lwu_23_apply_patches
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout exceeded (180 s) while flushing system log
-  check_types:
-  - tsan
-- name: 03100_lwu_deletes_4_index
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout exceeded (180 s) while flushing system log
-  check_types:
-  - tsan
-- name: 00284_external_aggregation
-  reason: INVESTIGATE - Timeout in query on tsan, msan
-  message: 'DB::Exception: Timeout exceeded'
-  check_types:
-  - tsan
-  - msan
-- name: 03550_analyzer_remote_view_columns
-  reason: INVESTIGATE - Timeout on tsan
-  message: 'DB::Exception: Timeout exceeded (180 s) while flushing system log'
-  check_types:
-  - tsan
-- name: 01630_simple_aggregate_all_functions_in_aggregating_merge_tree
-  reason: INVESTIGATE - Timeout on tsan
-  message: Timeout exceeded
-  check_types:
-  - tsan
-- name: 03593_backup_with_broken_projection
-  reason: INVESTIGATE - Timeout on msan
-  message: Timeout
-  check_types:
-  - msan
-- name: 00159_parallel_formatting_http
-  reason: INVESTIGATE - Timeout in curl on tsan
-  message: 'curl: (28) Operation timed out'
-  check_types:
-  - tsan
 - name: 02676_optimize_old_parts_replicated
   reason: INVESTIGATE - Unstable on tsan
   message: result differs with reference
@@ -379,24 +136,37 @@
   - tsan
 - name: test_database_hms/test.py::test_list_tables
   reason: 'KNOWN: Occasional timeout'
-  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed
-    out after 300 seconds
+  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed out after 300 seconds
 - name: test_database_hms/test.py::test_many_namespaces
   reason: 'KNOWN: Occasional timeout'
-  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed
-    out after 300 seconds
+  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed out after 300 seconds
 - name: test_database_hms/test.py::test_select
   reason: 'KNOWN: Occasional timeout'
-  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed
-    out after 300 seconds
+  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed out after 300 seconds
 - name: test_database_hms/test.py::test_tables_with_same_location
   reason: 'KNOWN: Occasional timeout'
-  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed
-    out after 300 seconds
+  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed out after 300 seconds
 - name: test_database_hms/test.py::test_hide_sensitive_info
   reason: 'KNOWN: Occasional timeout'
-  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed
-    out after 300 seconds
+  message: docker_compose_iceberg_hms_catalog.yml', '--verbose', 'up', '-d']' timed out after 300 seconds
 - name: test_keeper_memory_soft_limit/test.py::test_soft_limit_create
   reason: 'INVESTIGATE: Out of Memory'
   message: 'Coordination error: Out of Memory.'
+
+# Regex rules should be ordered from most specific to least specific.
+# regex: true applies to name, message, and not_message fields, but not to reason or check_types fields.
+
+# Match all sanitizer related timeouts in stateless tests
+- name: \d{5}_.*
+  reason: 'KNOWN: Random timeout with sanitizer'
+  message: 'Timeout! Processes left in process group
+    |DB::Exception: Timeout exceeded
+    |Timeout exceeded while receiving data from server
+    |curl: \(28\) Operation timed out'
+  not_message: 'Sanitizer trap'
+  regex: true
+  check_types:
+  - tsan
+  - asan
+  - msan
+  - ubsan