diff --git a/docs/changelog/134232.yaml b/docs/changelog/134232.yaml new file mode 100644 index 0000000000000..5e51297e7ff95 --- /dev/null +++ b/docs/changelog/134232.yaml @@ -0,0 +1,5 @@ +pr: 134232 +summary: Add relevant attributes to search took time APM metrics +area: Search +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java new file mode 100644 index 0000000000000..b0f5aeeb0e0f1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -0,0 +1,279 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.search; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; + +import java.util.HashMap; +import java.util.Map; + +/** + * Used to introspect a search request and extract metadata from it around the features it uses. + * Given that the purpose of this class is to extract metrics attributes, it should do its best + * to extract the minimum set of needed information without hurting performance, and without + * ever breaking: if something goes wrong around extracting attributes, it should skip extracting + * them as opposed to failing the search. + */ +public final class SearchRequestAttributesExtractor { + private static final Logger logger = LogManager.getLogger(SearchRequestAttributesExtractor.class); + + private SearchRequestAttributesExtractor() {} + + /** + * Introspects the provided search request and extracts metadata from it about some of its characteristics. + * + */ + public static Map extractAttributes(SearchRequest searchRequest, String[] localIndices) { + String target = extractIndices(localIndices); + + String pitOrScroll = null; + if (searchRequest.scroll() != null) { + pitOrScroll = SCROLL; + } + + SearchSourceBuilder searchSourceBuilder = searchRequest.source(); + if (searchSourceBuilder == null) { + return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll); + } + + if (searchSourceBuilder.pointInTimeBuilder() != null) { + pitOrScroll = PIT; + } + + final String primarySort; + if (searchSourceBuilder.sorts() == null || searchSourceBuilder.sorts().isEmpty()) { + primarySort = ScoreSortBuilder.NAME; + } else { + primarySort = extractPrimarySort(searchSourceBuilder.sorts().getFirst()); + } + + final String queryType = extractQueryType(searchSourceBuilder); + + QueryMetadataBuilder queryMetadataBuilder = new QueryMetadataBuilder(); + if (searchSourceBuilder.query() != null) { + try { + introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder, 0); + } catch (Exception e) { + logger.error("Failed to extract query attribute", e); + } + } + + final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; + return buildAttributesMap( + target, + primarySort, + queryType, + hasKnn, + queryMetadataBuilder.rangeOnTimestamp, + queryMetadataBuilder.rangeOnEventIngested, + pitOrScroll + ); + } + + private static Map buildAttributesMap( + String target, + String primarySort, + String queryType, + boolean knn, + boolean rangeOnTimestamp, + boolean rangeOnEventIngested, + String pitOrScroll + ) { + Map attributes = new HashMap<>(5, 1.0f); + attributes.put(TARGET_ATTRIBUTE, target); + attributes.put(SORT_ATTRIBUTE, primarySort); + attributes.put(QUERY_TYPE_ATTRIBUTE, queryType); + if (pitOrScroll != null) { + attributes.put(PIT_SCROLL_ATTRIBUTE, pitOrScroll); + } + if (knn) { + attributes.put(KNN_ATTRIBUTE, knn); + } + if (rangeOnTimestamp) { + attributes.put(RANGE_TIMESTAMP_ATTRIBUTE, rangeOnTimestamp); + } + if (rangeOnEventIngested) { + attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested); + } + return attributes; + } + + private static final class QueryMetadataBuilder { + private boolean knnQuery = false; + private boolean rangeOnTimestamp = false; + private boolean rangeOnEventIngested = false; + } + + static final String TARGET_ATTRIBUTE = "target"; + static final String SORT_ATTRIBUTE = "sort"; + static final String QUERY_TYPE_ATTRIBUTE = "query_type"; + static final String PIT_SCROLL_ATTRIBUTE = "pit_scroll"; + static final String KNN_ATTRIBUTE = "knn"; + static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp"; + static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested"; + + private static final String TARGET_KIBANA = ".kibana"; + private static final String TARGET_ML = ".ml"; + private static final String TARGET_FLEET = ".fleet"; + private static final String TARGET_SLO = ".slo"; + private static final String TARGET_ALERTS = ".alerts"; + private static final String TARGET_ELASTIC = ".elastic"; + private static final String TARGET_DS = ".ds-"; + private static final String TARGET_OTHERS = ".others"; + private static final String TARGET_USER = "user"; + private static final String ERROR = "error"; + + static String extractIndices(String[] indices) { + try { + // Note that indices are expected to be resolved, meaning wildcards are not handled on purpose + // If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices + if (indices.length == 1) { + String index = indices[0]; + assert Regex.isSimpleMatchPattern(index) == false; + if (index.startsWith(".")) { + if (index.startsWith(TARGET_KIBANA)) { + return TARGET_KIBANA; + } + if (index.startsWith(TARGET_ML)) { + return TARGET_ML; + } + if (index.startsWith(TARGET_FLEET)) { + return TARGET_FLEET; + } + if (index.startsWith(TARGET_SLO)) { + return TARGET_SLO; + } + if (index.startsWith(TARGET_ALERTS)) { + return TARGET_ALERTS; + } + if (index.startsWith(TARGET_ELASTIC)) { + return TARGET_ELASTIC; + } + // this happens only when data streams backing indices are searched explicitly + if (index.startsWith(TARGET_DS)) { + return TARGET_DS; + } + return TARGET_OTHERS; + } + } + return TARGET_USER; + } catch (Exception e) { + logger.error("Failed to extract indices attribute", e); + return ERROR; + } + } + + private static final String TIMESTAMP = "@timestamp"; + private static final String EVENT_INGESTED = "event.ingested"; + private static final String _DOC = "_doc"; + private static final String FIELD = "field"; + + static String extractPrimarySort(SortBuilder primarySortBuilder) { + try { + if (primarySortBuilder instanceof FieldSortBuilder fieldSort) { + return switch (fieldSort.getFieldName()) { + case TIMESTAMP -> TIMESTAMP; + case EVENT_INGESTED -> EVENT_INGESTED; + case _DOC -> _DOC; + default -> FIELD; + }; + } + return primarySortBuilder.getWriteableName(); + } catch (Exception e) { + logger.error("Failed to extract primary sort attribute", e); + return ERROR; + } + } + + private static final String HITS_AND_AGGS = "hits_and_aggs"; + private static final String HITS_ONLY = "hits_only"; + private static final String AGGS_ONLY = "aggs_only"; + private static final String COUNT_ONLY = "count_only"; + private static final String PIT = "pit"; + private static final String SCROLL = "scroll"; + + public static final Map SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL); + + static String extractQueryType(SearchSourceBuilder searchSourceBuilder) { + try { + int size = searchSourceBuilder.size(); + if (size == -1) { + size = SearchService.DEFAULT_SIZE; + } + if (searchSourceBuilder.aggregations() != null && size > 0) { + return HITS_AND_AGGS; + } + if (searchSourceBuilder.aggregations() != null) { + return AGGS_ONLY; + } + if (size > 0) { + return HITS_ONLY; + } + return COUNT_ONLY; + } catch (Exception e) { + logger.error("Failed to extract query type attribute", e); + return ERROR; + } + } + + private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder, int level) { + if (level > 20) { + return; + } + switch (queryBuilder) { + case BoolQueryBuilder bool: + for (QueryBuilder must : bool.must()) { + introspectQueryBuilder(must, queryMetadataBuilder, ++level); + } + for (QueryBuilder filter : bool.filter()) { + introspectQueryBuilder(filter, queryMetadataBuilder, ++level); + } + if (bool.must().isEmpty() && bool.filter().isEmpty() && bool.mustNot().isEmpty() && bool.should().size() == 1) { + introspectQueryBuilder(bool.should().getFirst(), queryMetadataBuilder, ++level); + } + // Note that should clauses are ignored unless there's only one that becomes mandatory + // must_not clauses are also ignored for now + break; + case ConstantScoreQueryBuilder constantScore: + introspectQueryBuilder(constantScore.innerQuery(), queryMetadataBuilder, ++level); + break; + case BoostingQueryBuilder boosting: + introspectQueryBuilder(boosting.positiveQuery(), queryMetadataBuilder, ++level); + break; + case NestedQueryBuilder nested: + introspectQueryBuilder(nested.query(), queryMetadataBuilder, ++level); + break; + case RangeQueryBuilder range: + switch (range.fieldName()) { + case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true; + case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true; + } + break; + case KnnVectorQueryBuilder knn: + queryMetadataBuilder.knnQuery = true; + break; + default: + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 8c5e861c199b6..6366916c0a64f 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -424,8 +424,12 @@ public void onFailure(Exception e) { final ActionListener searchResponseActionListener; if (collectSearchTelemetry) { + Map searchRequestAttributes = SearchRequestAttributesExtractor.extractAttributes( + original, + Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new) + ); if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) { - searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics); + searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes); } else { CCSUsage.Builder usageBuilder = new CCSUsage.Builder(); usageBuilder.setRemotesCount(resolvedIndices.getRemoteClusterIndices().size()); @@ -450,7 +454,13 @@ public void onFailure(Exception e) { if (shouldMinimizeRoundtrips(rewritten)) { usageBuilder.setFeature(CCSUsageTelemetry.MRT_FEATURE); } - searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, usageService, usageBuilder); + searchResponseActionListener = new SearchTelemetryListener( + delegate, + searchResponseMetrics, + searchRequestAttributes, + usageService, + usageBuilder + ); } } else { searchResponseActionListener = delegate; @@ -2035,23 +2045,31 @@ private static class SearchTelemetryListener extends DelegatingActionListener searchRequestAttributes; SearchTelemetryListener( ActionListener listener, SearchResponseMetrics searchResponseMetrics, + Map searchRequestAttributes, UsageService usageService, CCSUsage.Builder usageBuilder ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; + this.searchRequestAttributes = searchRequestAttributes; this.collectCCSTelemetry = true; this.usageService = usageService; this.usageBuilder = usageBuilder; } - SearchTelemetryListener(ActionListener listener, SearchResponseMetrics searchResponseMetrics) { + SearchTelemetryListener( + ActionListener listener, + SearchResponseMetrics searchResponseMetrics, + Map searchRequestAttributes + ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; + this.searchRequestAttributes = searchRequestAttributes; this.collectCCSTelemetry = false; this.usageService = null; this.usageBuilder = null; @@ -2060,7 +2078,7 @@ private static class SearchTelemetryListener extends DelegatingActionListener 0) { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java index b232cd16ba65e..c83a72f55059e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java @@ -60,7 +60,7 @@ protected void doExecute(Task task, SearchScrollRequest request, ActionListener< @Override public void onResponse(SearchResponse searchResponse) { try { - searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); + searchResponseMetrics.recordTookTimeForSearchScroll(searchResponse.getTookInMillis()); SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index bec0ddb17f6ab..5df6153f766df 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest.action.search; +import org.elasticsearch.action.search.SearchRequestAttributesExtractor; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -66,8 +67,13 @@ private SearchResponseMetrics(LongHistogram tookDurationTotalMillisHistogram, Lo this.responseCountTotalCounter = responseCountTotalCounter; } - public long recordTookTime(long tookTime) { - tookDurationTotalMillisHistogram.record(tookTime); + public long recordTookTimeForSearchScroll(long tookTime) { + tookDurationTotalMillisHistogram.record(tookTime, SearchRequestAttributesExtractor.SEARCH_SCROLL_ATTRIBUTES); + return tookTime; + } + + public long recordTookTime(long tookTime, Map attributes) { + tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java new file mode 100644 index 0000000000000..ceb7e8cc9e8de --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -0,0 +1,350 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.search; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.GeoDistanceSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +public class SearchRequestAttributesExtractorTests extends ESTestCase { + + public void testIndexIntrospectionSingleIndex() { + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_task_manager" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_9_2_0" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_ingest_9_2_0" })); + assertEquals(".kibana", SearchRequestAttributesExtractor.extractIndices(new String[] { ".kibana_security_solution" })); + assertEquals(".fleet", SearchRequestAttributesExtractor.extractIndices(new String[] { ".fleet-agents" })); + assertEquals(".ml", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ml-anomalies" })); + assertEquals(".ml", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ml-notifications" })); + assertEquals(".slo", SearchRequestAttributesExtractor.extractIndices(new String[] { ".slo" })); + assertEquals(".alerts", SearchRequestAttributesExtractor.extractIndices(new String[] { ".alerts" })); + assertEquals(".elastic", SearchRequestAttributesExtractor.extractIndices(new String[] { ".elastic" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test1" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test2" })); + assertEquals(".ds-", SearchRequestAttributesExtractor.extractIndices(new String[] { ".ds-test3" })); + + assertEquals(".others", SearchRequestAttributesExtractor.extractIndices(new String[] { ".a" })); + assertEquals(".others", SearchRequestAttributesExtractor.extractIndices(new String[] { ".abcde" })); + + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "a" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "ab" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "abc" })); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { "abcd" })); + + String indexName = "a" + randomAlphaOfLengthBetween(3, 10); + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(new String[] { indexName })); + } + + public void testIndexIntrospectionMultipleIndices() { + int length = randomIntBetween(2, 10); + String[] indices = new String[length]; + for (int i = 0; i < length; i++) { + indices[i] = randomAlphaOfLengthBetween(3, 10); + } + assertEquals("user", SearchRequestAttributesExtractor.extractIndices(indices)); + } + + public void testPrimarySortIntrospection() { + assertEquals("@timestamp", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("@timestamp"))); + assertEquals("event.ingested", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("event.ingested"))); + assertEquals("_doc", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder("_doc"))); + assertEquals("field", SearchRequestAttributesExtractor.extractPrimarySort(new FieldSortBuilder(randomAlphaOfLengthBetween(3, 10)))); + assertEquals("_score", SearchRequestAttributesExtractor.extractPrimarySort(new ScoreSortBuilder())); + assertEquals( + "_geo_distance", + SearchRequestAttributesExtractor.extractPrimarySort(new GeoDistanceSortBuilder(randomAlphaOfLengthBetween(3, 10), 1d, 1d)) + ); + assertEquals( + "_script", + SearchRequestAttributesExtractor.extractPrimarySort( + new ScriptSortBuilder(new Script("id"), randomFrom(ScriptSortBuilder.ScriptSortType.values())) + ) + ); + } + + public void testQueryTypeIntrospection() { + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + assertEquals("hits_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + assertEquals("hits_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + assertEquals("count_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(randomIntBetween(1, 100)); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("hits_and_aggs", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.size(0); + searchSourceBuilder.aggregation(new TermsAggregationBuilder("test")); + assertEquals("aggs_only", SearchRequestAttributesExtractor.extractQueryType(searchSourceBuilder)); + } + } + + private static void assertAttributes( + Map attributes, + String target, + String primarySort, + String queryType, + boolean knn, + boolean rangeOnTimestamp, + boolean rangeOnEventIngested, + String pitOrScroll + ) { + assertEquals(target, attributes.get(SearchRequestAttributesExtractor.TARGET_ATTRIBUTE)); + assertEquals(primarySort, attributes.get(SearchRequestAttributesExtractor.SORT_ATTRIBUTE)); + assertEquals(queryType, attributes.get(SearchRequestAttributesExtractor.QUERY_TYPE_ATTRIBUTE)); + assertEquals(pitOrScroll, attributes.get(SearchRequestAttributesExtractor.PIT_SCROLL_ATTRIBUTE)); + if (knn) { + assertEquals(knn, attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); + } else { + assertNull(attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); + } + if (rangeOnTimestamp) { + assertEquals(rangeOnTimestamp, attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); + } else { + assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); + } + if (rangeOnEventIngested) { + assertEquals(rangeOnEventIngested, attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); + } else { + assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); + } + } + + public void testExtractAttributes() { + { + SearchRequest searchRequest = new SearchRequest(); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "pit"); + } + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.scroll(new TimeValue(randomIntBetween(1, 10))); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, "scroll"); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.must(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + int numBool = randomIntBetween(2, 10); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (int i = 0; i < numBool; i++) { + BoolQueryBuilder boolQueryBuilderNew = new BoolQueryBuilder(); + boolQueryBuilder.filter(boolQueryBuilderNew); + boolQueryBuilder = boolQueryBuilderNew; + } + if (randomBoolean()) { + boolQueryBuilder.should(new RangeQueryBuilder("event.ingested")); + } + + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + boolQueryBuilder.must(new RangeQueryBuilder("event.ingested")); + boolQueryBuilder.must(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, true, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp")); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder(randomAlphaOfLengthBetween(3, 10))); + searchSourceBuilder.query(boolQueryBuilder); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, false, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new ConstantScoreQueryBuilder(new RangeQueryBuilder("@timestamp"))); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest(randomAlphaOfLengthBetween(3, 10)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.sort("@timestamp"); + searchSourceBuilder.query(new BoostingQueryBuilder(new RangeQueryBuilder("@timestamp"), new MatchAllQueryBuilder())); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "@timestamp", "hits_only", false, true, false, null); + } + } + + public void testDepthLimit() { + { + SearchRequest searchRequest = new SearchRequest("index"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(newBoolQueryBuilder); + int depth = randomIntBetween(5, 18); + for (int i = 0; i < depth; i++) { + BoolQueryBuilder innerBoolQueryBuilder = new BoolQueryBuilder(); + newBoolQueryBuilder.must(innerBoolQueryBuilder); + newBoolQueryBuilder = innerBoolQueryBuilder; + } + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, true, false, null); + } + { + SearchRequest searchRequest = new SearchRequest("index"); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); + BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(newBoolQueryBuilder); + int depth = randomIntBetween(19, 50); + for (int i = 0; i < depth; i++) { + BoolQueryBuilder innerBoolQueryBuilder = new BoolQueryBuilder(); + newBoolQueryBuilder.must(innerBoolQueryBuilder); + newBoolQueryBuilder = innerBoolQueryBuilder; + } + newBoolQueryBuilder.must(new RangeQueryBuilder("@timestamp")); + Map stringObjectMap = SearchRequestAttributesExtractor.extractAttributes( + searchRequest, + searchRequest.indices() + ); + assertAttributes(stringObjectMap, "user", "_score", "hits_only", false, false, false, null); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 3d9797a5f5f48..792b676f6002b 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -9,16 +9,22 @@ package org.elasticsearch.search.TelemetryMetrics; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.retriever.RescorerRetrieverBuilder; @@ -34,6 +40,8 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; @@ -52,7 +60,7 @@ protected boolean resetNodeAfterTest() { @Before public void setUpIndex() { - var num_primaries = randomIntBetween(1, 4); + var num_primaries = randomIntBetween(2, 4); createIndex( indexName, Settings.builder() @@ -61,9 +69,8 @@ public void setUpIndex() { .build() ); ensureGreen(indexName); - - prepareIndex(indexName).setId("1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -76,6 +83,133 @@ protected Collection> getPlugins() { return pluginList(TestTelemetryPlugin.class); } + public void testOthersDottedIndexName() { + createIndex(".whatever"); + createIndex(".kibana"); + { + SearchResponse searchResponse = client().prepareSearch(".whatever").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + assertEquals(".others", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(2, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + assertEquals(".kibana", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch(".*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(3, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // two dotted indices: categorized as "user" + assertSimpleQueryAttributes(measurement.attributes()); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana", ".whatever").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(4, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // two dotted indices: categorized as "user" + assertSimpleQueryAttributes(measurement.attributes()); + } + { + SearchResponse searchResponse = client().prepareSearch(".kibana", ".does_not_exist") + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .setQuery(simpleQueryStringQuery("foo")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(5, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(3, attributes.size()); + // because the second index does not exist, yet the search goes through, the remaining index is categorized correctly + assertEquals(".kibana", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } + { + SearchResponse searchResponse = client().prepareSearch("_all").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(6, measurements.size()); + Measurement measurement = measurements.getLast(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + } + + public void testIndexNameMustExist() { + SearchResponse searchResponse = client().prepareSearch(".must_exist") + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) + .setQuery(simpleQueryStringQuery("foo")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + } finally { + searchResponse.decRef(); + } + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // edge case rather than under .others (as it's a dotted index name), the index is categorized under "user" because no existing + // indices are targeted. + assertSimpleQueryAttributes(measurement.attributes()); + } + public void testSimpleQuery() { SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(simpleQueryStringQuery("foo")).get(); try { @@ -87,7 +221,55 @@ public void testSimpleQuery() { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); assertEquals(1, measurements.size()); - assertEquals(searchResponse.getTook().millis(), measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testSimpleQueryAgainstWildcardExpression() { + SearchResponse searchResponse = client().prepareSearch("*").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testSimpleQueryAgainstAlias() { + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest( + RestUtils.REST_MASTER_TIMEOUT_DEFAULT, + new TimeValue(30, TimeUnit.SECONDS) + ); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add().indices(indexName).alias(".alias")); + IndicesAliasesResponse indicesAliasesResponse = client().admin().indices().aliases(indicesAliasesRequest).actionGet(); + assertFalse(indicesAliasesResponse.hasErrors()); + SearchResponse searchResponse = client().prepareSearch(".alias").setQuery(simpleQueryStringQuery("foo")).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + private static void assertSimpleQueryAttributes(Map attributes) { + assertEquals(3, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); } public void testCompoundRetriever() { @@ -141,6 +323,7 @@ public void testMultiSearch() { int i = 0; for (Measurement measurement : measurements) { assertEquals(tookTimes.get(i++).longValue(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); } } @@ -151,17 +334,106 @@ public void testScroll() { client().prepareSearch(indexName).setSize(1).setQuery(simpleQueryStringQuery("foo")), 2, (respNum, response) -> { - if (respNum <= 2) { + if (respNum == 1) { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement( TOOK_DURATION_TOTAL_HISTOGRAM_NAME ); assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("scroll", attributes.get("pit_scroll")); + assertEquals("_score", attributes.get("sort")); + } else { + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + TOOK_DURATION_TOTAL_HISTOGRAM_NAME + ); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + Map attributes = measurement.attributes(); + assertEquals(1, attributes.size()); + assertEquals("scroll", attributes.get("query_type")); } resetMeter(); } ); } + /** + * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute + */ + public void testTimeRangeFilterNoResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2025-01-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse); + // can match kicked in, query got rewritten to match_none, yet we extracted the time range before rewrite + assertEquals(searchResponse.getSkippedShards(), searchResponse.getTotalShards()); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + /** + * Make sure that despite can match and query rewrite, we see the time range filter and record its corresponding attribute + */ + public void testTimeRangeFilterAllResults() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + public void testTimeRangeFilterOneResult() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-12-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertTimeRangeAttributes(measurement.attributes()); + } + + private static void assertTimeRangeAttributes(Map attributes) { + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + } + private void resetMeter() { getTestTelemetryPlugin().resetMeter(); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index f3dbc9ac67202..e0d6189d15a8a 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; @@ -95,7 +96,8 @@ public void testAsyncForegroundQuery() { } final List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); assertEquals(1, measurements.size()); - assertEquals(tookInMillis, measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(tookInMillis, measurement.getLong()); for (int i = 0; i < randomIntBetween(3, 10); i++) { AsyncSearchResponse asyncSearchResponse2 = client().execute(GetAsyncSearchAction.INSTANCE, new GetAsyncResultRequest(id)) @@ -106,7 +108,8 @@ public void testAsyncForegroundQuery() { asyncSearchResponse2.decRef(); } assertEquals(1, measurements.size()); - assertEquals(tookInMillis, measurements.getFirst().getLong()); + assertEquals(tookInMillis, measurement.getLong()); + assertAttributes(measurement.attributes()); } } @@ -142,7 +145,9 @@ public void testAsyncBackgroundQuery() throws Exception { SearchResponse searchResponse = asyncSearchResponse2.getSearchResponse(); try { assertSearchHits(searchResponse, "1", "2"); - assertEquals(searchResponse.getTook().millis(), measurements.getFirst().getLong()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertAttributes(measurement.attributes()); } finally { asyncSearchResponse2.decRef(); } @@ -152,4 +157,11 @@ public void testAsyncBackgroundQuery() throws Exception { private TestTelemetryPlugin getTestTelemetryPlugin() { return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().getFirst(); } + + private static void assertAttributes(Map attributes) { + assertEquals(3, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + } }