Skip to content

Commit 136321f

Browse files
javannajavanna
authored
Add relevant attributes to search took time APM metrics (#134232)
We already record the took time of a search request via took metric. We'd like to be able to slice such latencies based on some recurring categories of the request: - does it have agg or hit only? - is it sorted by field or by score? - does it have a time range filter? - does it target user data or internal indices? This commit introduces introspection for a search request and stores the extracted attributes together with the search took time metric.
1 parent 22f2d72 commit 136321f

File tree

8 files changed

+958
-16
lines changed

8 files changed

+958
-16
lines changed

‎docs/changelog/134232.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134232
2+
summary: Add relevant attributes to search took time APM metrics
3+
area: Search
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.action.search;
11+
12+
import org.apache.logging.log4j.LogManager;
13+
import org.apache.logging.log4j.Logger;
14+
import org.elasticsearch.common.regex.Regex;
15+
import org.elasticsearch.index.query.BoolQueryBuilder;
16+
import org.elasticsearch.index.query.BoostingQueryBuilder;
17+
import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
18+
import org.elasticsearch.index.query.NestedQueryBuilder;
19+
import org.elasticsearch.index.query.QueryBuilder;
20+
import org.elasticsearch.index.query.RangeQueryBuilder;
21+
import org.elasticsearch.search.SearchService;
22+
import org.elasticsearch.search.builder.SearchSourceBuilder;
23+
import org.elasticsearch.search.sort.FieldSortBuilder;
24+
import org.elasticsearch.search.sort.ScoreSortBuilder;
25+
import org.elasticsearch.search.sort.SortBuilder;
26+
import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
27+
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
/**
32+
* Used to introspect a search request and extract metadata from it around the features it uses.
33+
* Given that the purpose of this class is to extract metrics attributes, it should do its best
34+
* to extract the minimum set of needed information without hurting performance, and without
35+
* ever breaking: if something goes wrong around extracting attributes, it should skip extracting
36+
* them as opposed to failing the search.
37+
*/
38+
public final class SearchRequestAttributesExtractor {
39+
private static final Logger logger = LogManager.getLogger(SearchRequestAttributesExtractor.class);
40+
41+
private SearchRequestAttributesExtractor() {}
42+
43+
/**
44+
* Introspects the provided search request and extracts metadata from it about some of its characteristics.
45+
*
46+
*/
47+
public static Map<String, Object> extractAttributes(SearchRequest searchRequest, String[] localIndices) {
48+
String target = extractIndices(localIndices);
49+
50+
String pitOrScroll = null;
51+
if (searchRequest.scroll() != null) {
52+
pitOrScroll = SCROLL;
53+
}
54+
55+
SearchSourceBuilder searchSourceBuilder = searchRequest.source();
56+
if (searchSourceBuilder == null) {
57+
return buildAttributesMap(target, ScoreSortBuilder.NAME, HITS_ONLY, false, false, false, pitOrScroll);
58+
}
59+
60+
if (searchSourceBuilder.pointInTimeBuilder() != null) {
61+
pitOrScroll = PIT;
62+
}
63+
64+
final String primarySort;
65+
if (searchSourceBuilder.sorts() == null || searchSourceBuilder.sorts().isEmpty()) {
66+
primarySort = ScoreSortBuilder.NAME;
67+
} else {
68+
primarySort = extractPrimarySort(searchSourceBuilder.sorts().getFirst());
69+
}
70+
71+
final String queryType = extractQueryType(searchSourceBuilder);
72+
73+
QueryMetadataBuilder queryMetadataBuilder = new QueryMetadataBuilder();
74+
if (searchSourceBuilder.query() != null) {
75+
try {
76+
introspectQueryBuilder(searchSourceBuilder.query(), queryMetadataBuilder, 0);
77+
} catch (Exception e) {
78+
logger.error("Failed to extract query attribute", e);
79+
}
80+
}
81+
82+
final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery;
83+
return buildAttributesMap(
84+
target,
85+
primarySort,
86+
queryType,
87+
hasKnn,
88+
queryMetadataBuilder.rangeOnTimestamp,
89+
queryMetadataBuilder.rangeOnEventIngested,
90+
pitOrScroll
91+
);
92+
}
93+
94+
private static Map<String, Object> buildAttributesMap(
95+
String target,
96+
String primarySort,
97+
String queryType,
98+
boolean knn,
99+
boolean rangeOnTimestamp,
100+
boolean rangeOnEventIngested,
101+
String pitOrScroll
102+
) {
103+
Map<String, Object> attributes = new HashMap<>(5, 1.0f);
104+
attributes.put(TARGET_ATTRIBUTE, target);
105+
attributes.put(SORT_ATTRIBUTE, primarySort);
106+
attributes.put(QUERY_TYPE_ATTRIBUTE, queryType);
107+
if (pitOrScroll != null) {
108+
attributes.put(PIT_SCROLL_ATTRIBUTE, pitOrScroll);
109+
}
110+
if (knn) {
111+
attributes.put(KNN_ATTRIBUTE, knn);
112+
}
113+
if (rangeOnTimestamp) {
114+
attributes.put(RANGE_TIMESTAMP_ATTRIBUTE, rangeOnTimestamp);
115+
}
116+
if (rangeOnEventIngested) {
117+
attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested);
118+
}
119+
return attributes;
120+
}
121+
122+
private static final class QueryMetadataBuilder {
123+
private boolean knnQuery = false;
124+
private boolean rangeOnTimestamp = false;
125+
private boolean rangeOnEventIngested = false;
126+
}
127+
128+
static final String TARGET_ATTRIBUTE = "target";
129+
static final String SORT_ATTRIBUTE = "sort";
130+
static final String QUERY_TYPE_ATTRIBUTE = "query_type";
131+
static final String PIT_SCROLL_ATTRIBUTE = "pit_scroll";
132+
static final String KNN_ATTRIBUTE = "knn";
133+
static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp";
134+
static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested";
135+
136+
private static final String TARGET_KIBANA = ".kibana";
137+
private static final String TARGET_ML = ".ml";
138+
private static final String TARGET_FLEET = ".fleet";
139+
private static final String TARGET_SLO = ".slo";
140+
private static final String TARGET_ALERTS = ".alerts";
141+
private static final String TARGET_ELASTIC = ".elastic";
142+
private static final String TARGET_DS = ".ds-";
143+
private static final String TARGET_OTHERS = ".others";
144+
private static final String TARGET_USER = "user";
145+
private static final String ERROR = "error";
146+
147+
static String extractIndices(String[] indices) {
148+
try {
149+
// Note that indices are expected to be resolved, meaning wildcards are not handled on purpose
150+
// If indices resolve to data streams, the name of the data stream is returned as opposed to its backing indices
151+
if (indices.length == 1) {
152+
String index = indices[0];
153+
assert Regex.isSimpleMatchPattern(index) == false;
154+
if (index.startsWith(".")) {
155+
if (index.startsWith(TARGET_KIBANA)) {
156+
return TARGET_KIBANA;
157+
}
158+
if (index.startsWith(TARGET_ML)) {
159+
return TARGET_ML;
160+
}
161+
if (index.startsWith(TARGET_FLEET)) {
162+
return TARGET_FLEET;
163+
}
164+
if (index.startsWith(TARGET_SLO)) {
165+
return TARGET_SLO;
166+
}
167+
if (index.startsWith(TARGET_ALERTS)) {
168+
return TARGET_ALERTS;
169+
}
170+
if (index.startsWith(TARGET_ELASTIC)) {
171+
return TARGET_ELASTIC;
172+
}
173+
// this happens only when data streams backing indices are searched explicitly
174+
if (index.startsWith(TARGET_DS)) {
175+
return TARGET_DS;
176+
}
177+
return TARGET_OTHERS;
178+
}
179+
}
180+
return TARGET_USER;
181+
} catch (Exception e) {
182+
logger.error("Failed to extract indices attribute", e);
183+
return ERROR;
184+
}
185+
}
186+
187+
private static final String TIMESTAMP = "@timestamp";
188+
private static final String EVENT_INGESTED = "event.ingested";
189+
private static final String _DOC = "_doc";
190+
private static final String FIELD = "field";
191+
192+
static String extractPrimarySort(SortBuilder<?> primarySortBuilder) {
193+
try {
194+
if (primarySortBuilder instanceof FieldSortBuilder fieldSort) {
195+
return switch (fieldSort.getFieldName()) {
196+
case TIMESTAMP -> TIMESTAMP;
197+
case EVENT_INGESTED -> EVENT_INGESTED;
198+
case _DOC -> _DOC;
199+
default -> FIELD;
200+
};
201+
}
202+
return primarySortBuilder.getWriteableName();
203+
} catch (Exception e) {
204+
logger.error("Failed to extract primary sort attribute", e);
205+
return ERROR;
206+
}
207+
}
208+
209+
private static final String HITS_AND_AGGS = "hits_and_aggs";
210+
private static final String HITS_ONLY = "hits_only";
211+
private static final String AGGS_ONLY = "aggs_only";
212+
private static final String COUNT_ONLY = "count_only";
213+
private static final String PIT = "pit";
214+
private static final String SCROLL = "scroll";
215+
216+
public static final Map<String, Object> SEARCH_SCROLL_ATTRIBUTES = Map.of(QUERY_TYPE_ATTRIBUTE, SCROLL);
217+
218+
static String extractQueryType(SearchSourceBuilder searchSourceBuilder) {
219+
try {
220+
int size = searchSourceBuilder.size();
221+
if (size == -1) {
222+
size = SearchService.DEFAULT_SIZE;
223+
}
224+
if (searchSourceBuilder.aggregations() != null && size > 0) {
225+
return HITS_AND_AGGS;
226+
}
227+
if (searchSourceBuilder.aggregations() != null) {
228+
return AGGS_ONLY;
229+
}
230+
if (size > 0) {
231+
return HITS_ONLY;
232+
}
233+
return COUNT_ONLY;
234+
} catch (Exception e) {
235+
logger.error("Failed to extract query type attribute", e);
236+
return ERROR;
237+
}
238+
}
239+
240+
private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetadataBuilder queryMetadataBuilder, int level) {
241+
if (level > 20) {
242+
return;
243+
}
244+
switch (queryBuilder) {
245+
case BoolQueryBuilder bool:
246+
for (QueryBuilder must : bool.must()) {
247+
introspectQueryBuilder(must, queryMetadataBuilder, ++level);
248+
}
249+
for (QueryBuilder filter : bool.filter()) {
250+
introspectQueryBuilder(filter, queryMetadataBuilder, ++level);
251+
}
252+
if (bool.must().isEmpty() && bool.filter().isEmpty() && bool.mustNot().isEmpty() && bool.should().size() == 1) {
253+
introspectQueryBuilder(bool.should().getFirst(), queryMetadataBuilder, ++level);
254+
}
255+
// Note that should clauses are ignored unless there's only one that becomes mandatory
256+
// must_not clauses are also ignored for now
257+
break;
258+
case ConstantScoreQueryBuilder constantScore:
259+
introspectQueryBuilder(constantScore.innerQuery(), queryMetadataBuilder, ++level);
260+
break;
261+
case BoostingQueryBuilder boosting:
262+
introspectQueryBuilder(boosting.positiveQuery(), queryMetadataBuilder, ++level);
263+
break;
264+
case NestedQueryBuilder nested:
265+
introspectQueryBuilder(nested.query(), queryMetadataBuilder, ++level);
266+
break;
267+
case RangeQueryBuilder range:
268+
switch (range.fieldName()) {
269+
case TIMESTAMP -> queryMetadataBuilder.rangeOnTimestamp = true;
270+
case EVENT_INGESTED -> queryMetadataBuilder.rangeOnEventIngested = true;
271+
}
272+
break;
273+
case KnnVectorQueryBuilder knn:
274+
queryMetadataBuilder.knnQuery = true;
275+
break;
276+
default:
277+
}
278+
}
279+
}

‎server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -424,8 +424,12 @@ public void onFailure(Exception e) {
424424

425425
final ActionListener<SearchResponse> searchResponseActionListener;
426426
if (collectSearchTelemetry) {
427+
Map<String, Object> searchRequestAttributes = SearchRequestAttributesExtractor.extractAttributes(
428+
original,
429+
Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new)
430+
);
427431
if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) {
428-
searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics);
432+
searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes);
429433
} else {
430434
CCSUsage.Builder usageBuilder = new CCSUsage.Builder();
431435
usageBuilder.setRemotesCount(resolvedIndices.getRemoteClusterIndices().size());
@@ -450,7 +454,13 @@ public void onFailure(Exception e) {
450454
if (shouldMinimizeRoundtrips(rewritten)) {
451455
usageBuilder.setFeature(CCSUsageTelemetry.MRT_FEATURE);
452456
}
453-
searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, usageService, usageBuilder);
457+
searchResponseActionListener = new SearchTelemetryListener(
458+
delegate,
459+
searchResponseMetrics,
460+
searchRequestAttributes,
461+
usageService,
462+
usageBuilder
463+
);
454464
}
455465
} else {
456466
searchResponseActionListener = delegate;
@@ -2035,23 +2045,31 @@ private static class SearchTelemetryListener extends DelegatingActionListener<Se
20352045
private final SearchResponseMetrics searchResponseMetrics;
20362046
private final UsageService usageService;
20372047
private final boolean collectCCSTelemetry;
2048+
private final Map<String, Object> searchRequestAttributes;
20382049

20392050
SearchTelemetryListener(
20402051
ActionListener<SearchResponse> listener,
20412052
SearchResponseMetrics searchResponseMetrics,
2053+
Map<String, Object> searchRequestAttributes,
20422054
UsageService usageService,
20432055
CCSUsage.Builder usageBuilder
20442056
) {
20452057
super(listener);
20462058
this.searchResponseMetrics = searchResponseMetrics;
2059+
this.searchRequestAttributes = searchRequestAttributes;
20472060
this.collectCCSTelemetry = true;
20482061
this.usageService = usageService;
20492062
this.usageBuilder = usageBuilder;
20502063
}
20512064

2052-
SearchTelemetryListener(ActionListener<SearchResponse> listener, SearchResponseMetrics searchResponseMetrics) {
2065+
SearchTelemetryListener(
2066+
ActionListener<SearchResponse> listener,
2067+
SearchResponseMetrics searchResponseMetrics,
2068+
Map<String, Object> searchRequestAttributes
2069+
) {
20532070
super(listener);
20542071
this.searchResponseMetrics = searchResponseMetrics;
2072+
this.searchRequestAttributes = searchRequestAttributes;
20552073
this.collectCCSTelemetry = false;
20562074
this.usageService = null;
20572075
this.usageBuilder = null;
@@ -2060,7 +2078,7 @@ private static class SearchTelemetryListener extends DelegatingActionListener<Se
20602078
@Override
20612079
public void onResponse(SearchResponse searchResponse) {
20622080
try {
2063-
searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis());
2081+
searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis(), searchRequestAttributes);
20642082
SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus =
20652083
SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS;
20662084
if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {

‎server/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ protected void doExecute(Task task, SearchScrollRequest request, ActionListener<
6060
@Override
6161
public void onResponse(SearchResponse searchResponse) {
6262
try {
63-
searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis());
63+
searchResponseMetrics.recordTookTimeForSearchScroll(searchResponse.getTookInMillis());
6464
SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus =
6565
SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS;
6666
if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.rest.action.search;
1111

12+
import org.elasticsearch.action.search.SearchRequestAttributesExtractor;
1213
import org.elasticsearch.telemetry.metric.LongCounter;
1314
import org.elasticsearch.telemetry.metric.LongHistogram;
1415
import org.elasticsearch.telemetry.metric.MeterRegistry;
@@ -66,8 +67,13 @@ private SearchResponseMetrics(LongHistogram tookDurationTotalMillisHistogram, Lo
6667
this.responseCountTotalCounter = responseCountTotalCounter;
6768
}
6869

69-
public long recordTookTime(long tookTime) {
70-
tookDurationTotalMillisHistogram.record(tookTime);
70+
public long recordTookTimeForSearchScroll(long tookTime) {
71+
tookDurationTotalMillisHistogram.record(tookTime, SearchRequestAttributesExtractor.SEARCH_SCROLL_ATTRIBUTES);
72+
return tookTime;
73+
}
74+
75+
public long recordTookTime(long tookTime, Map<String, Object> attributes) {
76+
tookDurationTotalMillisHistogram.record(tookTime, attributes);
7177
return tookTime;
7278
}
7379

0 commit comments

Comments
 (0)