From fad3c42d08dea0e99d2b9f328391f087191449d9 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 17:41:18 +0200 Subject: [PATCH 01/30] Add time range bucketing attribute to APM took time latency metrics This is similar to #135524, but adding the attribute to the took time latency metric. That requires a bit of ceremony as the took time metric is recorded on the coordinating node, while the time range filter is parsed on each shard. We don't have mappings available on the coord node, which are needed to parse dates on the coord node. Thus we need to rely on date parsing done on the data nodes, which requires sending back the parsed value to the coord node, performing some simple reduction on it, and adding it back to the search response. --- .../CountOnlyQueryPhaseResultConsumer.java | 3 +- .../action/search/RankFeaturePhase.java | 3 +- .../action/search/SearchPhaseController.java | 17 ++- .../SearchRequestAttributesExtractor.java | 7 ++ .../action/search/SearchResponse.java | 25 ++++- .../action/search/SearchResponseMerger.java | 1 + .../action/search/SearchResponseSections.java | 11 +- .../action/search/TransportSearchAction.java | 25 ++++- .../index/mapper/DateFieldMapper.java | 10 +- .../index/query/QueryRewriteContext.java | 18 ++++ .../index/query/RangeQueryBuilder.java | 3 +- .../index/query/SearchExecutionContext.java | 20 ---- .../action/search/SearchResponseMetrics.java | 3 +- .../elasticsearch/search/SearchService.java | 3 + .../search/query/QueryPhase.java | 1 + .../search/query/QuerySearchResult.java | 17 +++ .../referable/timestamp_range_telemetry.csv | 1 + .../resources/transport/upper_bounds/9.2.csv | 2 +- .../action/search/ExpandSearchPhaseTests.java | 28 +++-- .../search/FetchLookupFieldsPhaseTests.java | 7 +- .../action/search/SearchAsyncActionTests.java | 1 + .../search/TransportSearchActionTests.java | 1 + .../SearchTookTimeTelemetryTests.java | 100 ++++++++++++++++-- .../search/SearchResponseUtils.java | 6 +- .../AsyncSearchTookTimeTelemetryTests.java | 24 +++-- 25 files changed, 270 insertions(+), 67 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv diff --git a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java index a454181bf47a1..241141ea79e94 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java @@ -86,7 +86,8 @@ public SearchPhaseController.ReducedQueryPhase reduce() throws Exception { 1, 0, 0, - results.isEmpty() + results.isEmpty(), + null ); if (progressListener != SearchProgressListener.NOOP) { progressListener.notifyFinalReduce( diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 25238711c5c1c..c0f1566b2b281 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -251,7 +251,8 @@ private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults( reducedQueryPhase.numReducePhases(), reducedQueryPhase.size(), reducedQueryPhase.from(), - reducedQueryPhase.isEmptyResult() + reducedQueryPhase.isEmptyResult(), + reducedQueryPhase.rangeTimestampFrom() ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 2df4c05722908..8c39aa56e7545 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -487,7 +487,8 @@ static ReducedQueryPhase reducedQueryPhase( numReducePhases, 0, 0, - true + true, + null ); } final List nonNullResults = new ArrayList<>(); @@ -516,6 +517,7 @@ static ReducedQueryPhase reducedQueryPhase( : Collections.emptyMap(); int from = 0; int size = 0; + Long rangeTimestampFrom = null; DocValueFormat[] sortValueFormats = null; for (QuerySearchResult result : nonNullResults) { from = result.from(); @@ -525,6 +527,10 @@ static ReducedQueryPhase reducedQueryPhase( sortValueFormats = result.sortValueFormats(); } + if (rangeTimestampFrom == null) { + rangeTimestampFrom = result.getRangeTimestampFrom(); + } + if (hasSuggest) { assert result.suggest() != null; for (Suggestion> suggestion : result.suggest()) { @@ -579,7 +585,8 @@ static ReducedQueryPhase reducedQueryPhase( numReducePhases, size, from, - false + false, + rangeTimestampFrom ); } @@ -662,7 +669,8 @@ public record ReducedQueryPhase( // the offset into the merged top hits int from, // true iff the query phase had no results. Otherwise false - boolean isEmptyResult + boolean isEmptyResult, + Long rangeTimestampFrom ) { public ReducedQueryPhase { @@ -683,7 +691,8 @@ public SearchResponseSections buildResponse(SearchHits hits, Collection attributes) { + if (timeRangeFrom != null) { + String timestampRangeFilter = introspectTimeRange(timeRangeFrom, nowInMillis); + attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter); + } + } + static String introspectTimeRange(long timeRangeFrom, long nowInMillis) { for (TimeRangeBucket value : TimeRangeBucket.values()) { if (timeRangeFrom >= nowInMillis - value.millis) { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 7b82116e5b447..9bade1f721f03 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.OriginalIndices; @@ -61,6 +62,8 @@ */ public class SearchResponse extends ActionResponse implements ChunkedToXContentObject { + public static final TransportVersion TIMESTAMP_RANGE_TELEMETRY = TransportVersion.fromName("timestamp_range_telemetry"); + // for cross-cluster scenarios where cluster names are shown in API responses, use this string // rather than empty string (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) we use internally public static final String LOCAL_CLUSTER_NAME_REPRESENTATION = "(local)"; @@ -87,6 +90,7 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO private final ShardSearchFailure[] shardFailures; private final Clusters clusters; private final long tookInMillis; + private final Long rangeTimestampFrom; private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted()); @@ -122,6 +126,11 @@ public SearchResponse(StreamInput in) throws IOException { tookInMillis = in.readVLong(); skippedShards = in.readVInt(); pointInTimeId = in.readOptionalBytesReference(); + if (in.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { + rangeTimestampFrom = in.readOptionalLong(); + } else { + rangeTimestampFrom = null; + } } public SearchResponse( @@ -155,6 +164,7 @@ public SearchResponse( tookInMillis, shardFailures, clusters, + null, null ); } @@ -185,7 +195,8 @@ public SearchResponse( tookInMillis, shardFailures, clusters, - pointInTimeId + pointInTimeId, + searchResponseSections.rangeTimestampFrom ); } @@ -204,7 +215,8 @@ public SearchResponse( long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters, - BytesReference pointInTimeId + BytesReference pointInTimeId, + Long rangeTimestampFrom ) { this.hits = hits; hits.incRef(); @@ -225,6 +237,7 @@ public SearchResponse( assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards; assert scrollId == null || pointInTimeId == null : "SearchResponse can't have both scrollId [" + scrollId + "] and searchContextId [" + pointInTimeId + "]"; + this.rangeTimestampFrom = rangeTimestampFrom; } @Override @@ -462,6 +475,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(tookInMillis); out.writeVInt(skippedShards); out.writeOptionalBytesReference(pointInTimeId); + if (out.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { + out.writeOptionalLong(rangeTimestampFrom); + } + } + + public Long getRangeTimestampFrom() { + return rangeTimestampFrom; } @Override @@ -1152,6 +1172,7 @@ public static SearchResponse empty(Supplier tookInMillisSupplier, Clusters tookInMillisSupplier.get(), ShardSearchFailure.EMPTY_ARRAY, clusters, + null, null ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java index bbb4ce45c47dc..da68d181145bd 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java @@ -232,6 +232,7 @@ public SearchResponse getMergedResponse(Clusters clusters) { tookInMillis, shardFailures, clusters, + null, null ); } finally { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java index c63fcf5c3c025..07bf7d99d6277 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -32,7 +32,8 @@ public class SearchResponseSections implements Releasable { false, null, null, - 1 + 1, + null ); public static final SearchResponseSections EMPTY_WITHOUT_TOTAL_HITS = new SearchResponseSections( SearchHits.EMPTY_WITHOUT_TOTAL_HITS, @@ -41,7 +42,8 @@ public class SearchResponseSections implements Releasable { false, null, null, - 1 + 1, + null ); protected final SearchHits hits; protected final InternalAggregations aggregations; @@ -50,6 +52,7 @@ public class SearchResponseSections implements Releasable { protected final boolean timedOut; protected final Boolean terminatedEarly; protected final int numReducePhases; + protected final Long rangeTimestampFrom; public SearchResponseSections( SearchHits hits, @@ -58,7 +61,8 @@ public SearchResponseSections( boolean timedOut, Boolean terminatedEarly, SearchProfileResults profileResults, - int numReducePhases + int numReducePhases, + Long rangeTimestampFrom ) { this.hits = hits; this.aggregations = aggregations; @@ -67,6 +71,7 @@ public SearchResponseSections( this.timedOut = timedOut; this.terminatedEarly = terminatedEarly; this.numReducePhases = numReducePhases; + this.rangeTimestampFrom = rangeTimestampFrom; } public final SearchHits hits() { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index fd14f54371a5e..6a28592012c76 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -430,7 +430,12 @@ public void onFailure(Exception e) { Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new) ); if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) { - searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes); + searchResponseActionListener = new SearchTelemetryListener( + delegate, + searchResponseMetrics, + searchRequestAttributes, + timeProvider.absoluteStartMillis() + ); } else { CCSUsage.Builder usageBuilder = new CCSUsage.Builder(); usageBuilder.setRemotesCount(resolvedIndices.getRemoteClusterIndices().size()); @@ -459,6 +464,7 @@ public void onFailure(Exception e) { delegate, searchResponseMetrics, searchRequestAttributes, + timeProvider.absoluteStartMillis(), usageService, usageBuilder ); @@ -776,7 +782,8 @@ public void onResponse(SearchResponse searchResponse) { timeProvider.buildTookInMillis(), searchResponse.getShardFailures(), clusters, - searchResponse.pointInTimeId() + searchResponse.pointInTimeId(), + searchResponse.getRangeTimestampFrom() ) ); } @@ -2046,6 +2053,7 @@ static String[] ignoreBlockedIndices(ProjectState projectState, String[] concret private static class SearchTelemetryListener extends DelegatingActionListener { private final CCSUsage.Builder usageBuilder; private final SearchResponseMetrics searchResponseMetrics; + private final long nowInMillis; private final UsageService usageService; private final boolean collectCCSTelemetry; private final Map searchRequestAttributes; @@ -2054,12 +2062,14 @@ private static class SearchTelemetryListener extends DelegatingActionListener listener, SearchResponseMetrics searchResponseMetrics, Map searchRequestAttributes, + long nowInMillis, UsageService usageService, CCSUsage.Builder usageBuilder ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; this.searchRequestAttributes = searchRequestAttributes; + this.nowInMillis = nowInMillis; this.collectCCSTelemetry = true; this.usageService = usageService; this.usageBuilder = usageBuilder; @@ -2068,11 +2078,13 @@ private static class SearchTelemetryListener extends DelegatingActionListener listener, SearchResponseMetrics searchResponseMetrics, - Map searchRequestAttributes + Map searchRequestAttributes, + long nowInMillis ) { super(listener); this.searchResponseMetrics = searchResponseMetrics; this.searchRequestAttributes = searchRequestAttributes; + this.nowInMillis = nowInMillis; this.collectCCSTelemetry = false; this.usageService = null; this.usageBuilder = null; @@ -2081,7 +2093,12 @@ private static class SearchTelemetryListener extends DelegatingActionListener 0) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0e34261baa996..b74de0798e563 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -944,7 +944,7 @@ public Relation isFieldWithinQuery( minValue = Long.min(minValue, skipper.minValue()); maxValue = Long.max(maxValue, skipper.maxValue()); } - return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context); + return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name()); } byte[] minPackedValue = PointValues.getMinPackedValue(reader, name()); if (minPackedValue == null) { @@ -954,7 +954,7 @@ public Relation isFieldWithinQuery( long minValue = LongPoint.decodeDimension(minPackedValue, 0); long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0); - return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context); + return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name()); } public DateMathParser resolveDateMathParser(DateMathParser dateParser, Object from, Object to) { @@ -977,7 +977,8 @@ public Relation isFieldWithinQuery( boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, - QueryRewriteContext context + QueryRewriteContext context, + String fieldName ) { dateParser = resolveDateMathParser(dateParser, from, to); @@ -990,6 +991,9 @@ public Relation isFieldWithinQuery( } ++fromInclusive; } + if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { + context.setRangeTimestampFrom(fromInclusive); + } } long toInclusive = Long.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 1fc01915b7aec..faca20858b586 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -78,6 +78,7 @@ public class QueryRewriteContext { private QueryRewriteInterceptor queryRewriteInterceptor; private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; + private Long rangeTimestampFrom; public QueryRewriteContext( final XContentParserConfiguration parserConfiguration, @@ -520,4 +521,21 @@ public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInter this.queryRewriteInterceptor = queryRewriteInterceptor; } + /** + * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query + */ + public Long getRangeTimestampFrom() { + return rangeTimestampFrom; + } + + /** + * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. + */ + public void setRangeTimestampFrom(long rangeTimestampFrom) { + if (this.rangeTimestampFrom == null) { + this.rangeTimestampFrom = rangeTimestampFrom; + } else { + this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index c1787cf0a84cc..95b117bcbc311 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -448,7 +448,8 @@ protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext c includeUpper, timeZone, dateMathParser, - coordinatorRewriteContext + coordinatorRewriteContext, + dateFieldType.name() ); } // If the field type is null or not of type DataFieldType then we have no idea whether this range query will match during diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 1e8bd76ae03b8..6919bd725133c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -109,8 +109,6 @@ public class SearchExecutionContext extends QueryRewriteContext { private final Integer requestSize; private final MapperMetrics mapperMetrics; - private Long rangeTimestampFrom; - /** * Build a {@linkplain SearchExecutionContext}. */ @@ -745,22 +743,4 @@ public void setRewriteToNamedQueries() { public boolean rewriteToNamedQuery() { return rewriteToNamedQueries; } - - /** - * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query - */ - public Long getRangeTimestampFrom() { - return rangeTimestampFrom; - } - - /** - * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. - */ - public void setRangeTimestampFrom(Long rangeTimestampFrom) { - if (this.rangeTimestampFrom == null) { - this.rangeTimestampFrom = rangeTimestampFrom; - } else { - this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); - } - } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 5df6153f766df..10d9e065e1b4f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -72,7 +72,8 @@ public long recordTookTimeForSearchScroll(long tookTime) { return tookTime; } - public long recordTookTime(long tookTime, Map attributes) { + public long recordTookTime(long tookTime, Long rangeTimestampFrom, long nowInMillis, Map attributes) { + SearchRequestAttributesExtractor.addTimeRangeAttribute(rangeTimestampFrom, nowInMillis, attributes); tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 4c26be97da1b6..c34259e936c73 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1442,6 +1442,9 @@ private DefaultSearchContext createSearchContext( // during rewrite and normalized / evaluate templates etc. SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); + if (context.getRangeTimestampFrom() != null) { + searchContext.getSearchExecutionContext().setRangeTimestampFrom(context.getRangeTimestampFrom()); + } assert searchContext.getSearchExecutionContext().isCacheable(); success = true; } finally { diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 5fcfb2b9766cd..b07fe857e634a 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -157,6 +157,7 @@ static void addCollectorsAndSearch(SearchContext searchContext) throws QueryPhas final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); + queryResult.setRangeTimestampFrom(searchContext.getSearchExecutionContext().getRangeTimestampFrom()); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 69ed8e72a6510..1af7cc7bb3f22 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -14,6 +14,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.io.stream.DelayableWriteable; import org.elasticsearch.common.io.stream.StreamInput; @@ -77,6 +78,8 @@ public final class QuerySearchResult extends SearchPhaseResult { private final SubscribableListener aggsContextReleased; + private Long rangeTimestampFrom; + public QuerySearchResult() { this(false); } @@ -453,6 +456,9 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del reduced = in.readBoolean(); } } + if (in.getTransportVersion().onOrAfter(SearchResponse.TIMESTAMP_RANGE_TELEMETRY)) { + rangeTimestampFrom = in.readOptionalLong(); + } success = true; } finally { if (success == false) { @@ -523,6 +529,9 @@ public void writeToNoId(StreamOutput out) throws IOException { if (versionSupportsBatchedExecution(out.getTransportVersion())) { out.writeBoolean(reduced); } + if (out.getTransportVersion().onOrAfter(SearchResponse.TIMESTAMP_RANGE_TELEMETRY)) { + out.writeOptionalLong(rangeTimestampFrom); + } } @Nullable @@ -575,4 +584,12 @@ private static boolean versionSupportsBatchedExecution(TransportVersion transpor return transportVersion.onOrAfter(TransportVersions.BATCHED_QUERY_PHASE_VERSION) || transportVersion.isPatchFrom(TransportVersions.BATCHED_QUERY_PHASE_VERSION_BACKPORT_8_X); } + + public Long getRangeTimestampFrom() { + return rangeTimestampFrom; + } + + public void setRangeTimestampFrom(Long rangeTimestampFrom) { + this.rangeTimestampFrom = rangeTimestampFrom; + } } diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv new file mode 100644 index 0000000000000..4b012195cac35 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -0,0 +1 @@ +9177000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 78180d915cd67..dc4de129c5ab9 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -roles_security_stats,9176000 +timestamp_range_telemetry,9177000 diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index a08dbc80e6b9c..a1353bc3ebcc3 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -99,7 +99,16 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { try ( - var sections = new SearchResponseSections(collapsedHits.get(innerHitNum), null, null, false, null, null, 1) + var sections = new SearchResponseSections( + collapsedHits.get(innerHitNum), + null, + null, + false, + null, + null, + 1, + null + ) ) { mockSearchPhaseContext.sendSearchResponse(sections, null); } @@ -126,7 +135,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ), null ); @@ -199,7 +209,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); @@ -237,7 +248,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ), null ); @@ -270,7 +282,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL ); SearchHits hits = SearchHits.empty(new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); - final SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, null, null, 1); + final SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, null, null, 1, null); ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); phase.run(); mockSearchPhaseContext.assertNoFailure(); @@ -322,7 +334,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, null); @@ -393,7 +406,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { ExpandSearchPhase phase = newExpandSearchPhase(mockSearchPhaseContext, searchResponseSections, new AtomicArray<>(0)); diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java index b03d2be165937..96b26ea2f6a34 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java @@ -54,7 +54,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL false, null, null, - 1 + 1, + null ) ) { FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null); @@ -129,6 +130,7 @@ void sendExecuteMultiSearch( randomNonNegativeLong(), ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters(), + null, null ), null @@ -193,7 +195,8 @@ void sendExecuteMultiSearch( false, null, null, - 1 + 1, + null ) ) { FetchLookupFieldsPhase phase = new FetchLookupFieldsPhase(searchPhaseContext, sections, null); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index afd3bee4c4ab8..71d671ba4d89c 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -784,6 +784,7 @@ public static class TestSearchResponse extends SearchResponse { 0L, ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY, + null, null ); } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 61aa05f703018..112befac30dd6 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -1059,6 +1059,7 @@ private static void resolveWithEmptySearchResponse(Tuple attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + // there were no results, and no shards queried, hence no range filter extracted from the query either } /** @@ -404,6 +437,8 @@ public void testTimeRangeFilterAllResults() { assertEquals(1, measurements.size()); Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + // in this case the range query gets rewritten to a range query with open bounds on the shards. Here we test that query rewrite + // is able to grab the parsed range filter and propagate it all the way to the search response assertTimeRangeAttributes(measurement.attributes()); } @@ -427,11 +462,64 @@ public void testTimeRangeFilterOneResult() { } private static void assertTimeRangeAttributes(Map attributes) { + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + } + + public void testTimeRangeFilterAllResultsFilterOnEventIngested() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("event.ingested").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); assertEquals(4, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_event_ingested")); + } + + public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(10)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) + .setQuery(boolQueryBuilder) + .addSort(new FieldSortBuilder("@timestamp")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("@timestamp", attributes.get("sort")); assertEquals(true, attributes.get("range_timestamp")); + assertEquals("15_minutes", attributes.get("timestamp_range_filter")); } private void resetMeter() { @@ -439,6 +527,6 @@ private void resetMeter() { } private TestTelemetryPlugin getTestTelemetryPlugin() { - return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().get(0); + return getInstanceFromNode(PluginsService.class).filterPlugins(TestTelemetryPlugin.class).toList().getFirst(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java index de94a81eb00c1..82906316a30d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java @@ -224,7 +224,8 @@ public SearchResponse build() { tookInMillis, shardFailures == null ? ShardSearchFailure.EMPTY_ARRAY : shardFailures.toArray(ShardSearchFailure[]::new), clusters, - pointInTimeId + pointInTimeId, + null ); } } @@ -418,7 +419,8 @@ public static SearchResponse parseInnerSearchResponse(XContentParser parser) thr tookInMillis, failures.toArray(ShardSearchFailure.EMPTY_ARRAY), clusters, - searchContextId + searchContextId, + null ); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index e0d6189d15a8a..4a87c42d9f2d2 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -11,6 +11,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -58,8 +60,8 @@ public void setUpIndex() { ); ensureGreen(indexName); - prepareIndex(indexName).setId("1").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); - prepareIndex(indexName).setId("2").setSource("body", "foo").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("1").setSource("body", "foo", "@timestamp", "2024-11-01").setRefreshPolicy(IMMEDIATE).get(); + prepareIndex(indexName).setId("2").setSource("body", "foo", "@timestamp", "2024-12-01").setRefreshPolicy(IMMEDIATE).get(); } @After @@ -77,9 +79,10 @@ protected Collection> getPlugins() { * a sync request given its execution will always be completed directly as submit async search returns. */ public void testAsyncForegroundQuery() { - SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest( - new SearchSourceBuilder().query(simpleQueryStringQuery("foo")) - ); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest(new SearchSourceBuilder().query(boolQueryBuilder)); asyncSearchRequest.setWaitForCompletionTimeout(TimeValue.timeValueSeconds(5)); asyncSearchRequest.setKeepOnCompletion(true); AsyncSearchResponse asyncSearchResponse = client().execute(SubmitAsyncSearchAction.INSTANCE, asyncSearchRequest).actionGet(); @@ -119,9 +122,10 @@ public void testAsyncForegroundQuery() { * without any influence from get async search polling happening around the same async search request. */ public void testAsyncBackgroundQuery() throws Exception { - SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest( - new SearchSourceBuilder().query(simpleQueryStringQuery("foo")) - ); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SubmitAsyncSearchRequest asyncSearchRequest = new SubmitAsyncSearchRequest(new SearchSourceBuilder().query(boolQueryBuilder)); asyncSearchRequest.setWaitForCompletionTimeout(TimeValue.ZERO); AsyncSearchResponse asyncSearchResponse = client().execute(SubmitAsyncSearchAction.INSTANCE, asyncSearchRequest).actionGet(); String id; @@ -159,9 +163,11 @@ private TestTelemetryPlugin getTestTelemetryPlugin() { } private static void assertAttributes(Map attributes) { - assertEquals(3, attributes.size()); + assertEquals(5, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); } } From e81fb6c920a7b424ac1ad8b151f707e382608fb3 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 18:01:32 +0200 Subject: [PATCH 02/30] iter --- .../action/search/SearchResponse.java | 24 ++++--------------- .../action/search/SearchResponseMerger.java | 1 - .../action/search/TransportSearchAction.java | 3 +-- .../search/query/QuerySearchResult.java | 7 +++--- .../search/FetchLookupFieldsPhaseTests.java | 1 - .../action/search/SearchAsyncActionTests.java | 1 - .../search/TransportSearchActionTests.java | 1 - .../search/SearchResponseUtils.java | 6 ++--- 8 files changed, 12 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 9bade1f721f03..ebb16f57aa576 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.search; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.OriginalIndices; @@ -62,8 +61,6 @@ */ public class SearchResponse extends ActionResponse implements ChunkedToXContentObject { - public static final TransportVersion TIMESTAMP_RANGE_TELEMETRY = TransportVersion.fromName("timestamp_range_telemetry"); - // for cross-cluster scenarios where cluster names are shown in API responses, use this string // rather than empty string (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) we use internally public static final String LOCAL_CLUSTER_NAME_REPRESENTATION = "(local)"; @@ -90,7 +87,8 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO private final ShardSearchFailure[] shardFailures; private final Clusters clusters; private final long tookInMillis; - private final Long rangeTimestampFrom; + // only used for telemetry purposes on the coordinating node, where the search response gets created + private transient Long rangeTimestampFrom; private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted()); @@ -126,11 +124,6 @@ public SearchResponse(StreamInput in) throws IOException { tookInMillis = in.readVLong(); skippedShards = in.readVInt(); pointInTimeId = in.readOptionalBytesReference(); - if (in.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { - rangeTimestampFrom = in.readOptionalLong(); - } else { - rangeTimestampFrom = null; - } } public SearchResponse( @@ -164,7 +157,6 @@ public SearchResponse( tookInMillis, shardFailures, clusters, - null, null ); } @@ -195,9 +187,9 @@ public SearchResponse( tookInMillis, shardFailures, clusters, - pointInTimeId, - searchResponseSections.rangeTimestampFrom + pointInTimeId ); + this.rangeTimestampFrom = searchResponseSections.rangeTimestampFrom; } public SearchResponse( @@ -215,8 +207,7 @@ public SearchResponse( long tookInMillis, ShardSearchFailure[] shardFailures, Clusters clusters, - BytesReference pointInTimeId, - Long rangeTimestampFrom + BytesReference pointInTimeId ) { this.hits = hits; hits.incRef(); @@ -237,7 +228,6 @@ public SearchResponse( assert skippedShards <= totalShards : "skipped: " + skippedShards + " total: " + totalShards; assert scrollId == null || pointInTimeId == null : "SearchResponse can't have both scrollId [" + scrollId + "] and searchContextId [" + pointInTimeId + "]"; - this.rangeTimestampFrom = rangeTimestampFrom; } @Override @@ -475,9 +465,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(tookInMillis); out.writeVInt(skippedShards); out.writeOptionalBytesReference(pointInTimeId); - if (out.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { - out.writeOptionalLong(rangeTimestampFrom); - } } public Long getRangeTimestampFrom() { @@ -1172,7 +1159,6 @@ public static SearchResponse empty(Supplier tookInMillisSupplier, Clusters tookInMillisSupplier.get(), ShardSearchFailure.EMPTY_ARRAY, clusters, - null, null ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java index da68d181145bd..bbb4ce45c47dc 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseMerger.java @@ -232,7 +232,6 @@ public SearchResponse getMergedResponse(Clusters clusters) { tookInMillis, shardFailures, clusters, - null, null ); } finally { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 6a28592012c76..a07d23fcd2ace 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -782,8 +782,7 @@ public void onResponse(SearchResponse searchResponse) { timeProvider.buildTookInMillis(), searchResponse.getShardFailures(), clusters, - searchResponse.pointInTimeId(), - searchResponse.getRangeTimestampFrom() + searchResponse.pointInTimeId() ) ); } diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 1af7cc7bb3f22..d1e4281904f7f 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -14,7 +14,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.io.stream.DelayableWriteable; import org.elasticsearch.common.io.stream.StreamInput; @@ -45,6 +44,8 @@ import static org.elasticsearch.common.lucene.Lucene.writeTopDocs; public final class QuerySearchResult extends SearchPhaseResult { + private static final TransportVersion TIMESTAMP_RANGE_TELEMETRY = TransportVersion.fromName("timestamp_range_telemetry"); + private int from; private int size; private TopDocsAndMaxScore topDocsAndMaxScore; @@ -456,7 +457,7 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del reduced = in.readBoolean(); } } - if (in.getTransportVersion().onOrAfter(SearchResponse.TIMESTAMP_RANGE_TELEMETRY)) { + if (in.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { rangeTimestampFrom = in.readOptionalLong(); } success = true; @@ -529,7 +530,7 @@ public void writeToNoId(StreamOutput out) throws IOException { if (versionSupportsBatchedExecution(out.getTransportVersion())) { out.writeBoolean(reduced); } - if (out.getTransportVersion().onOrAfter(SearchResponse.TIMESTAMP_RANGE_TELEMETRY)) { + if (out.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { out.writeOptionalLong(rangeTimestampFrom); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java index 96b26ea2f6a34..fe4259b582528 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchLookupFieldsPhaseTests.java @@ -130,7 +130,6 @@ void sendExecuteMultiSearch( randomNonNegativeLong(), ShardSearchFailure.EMPTY_ARRAY, SearchResponseTests.randomClusters(), - null, null ), null diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 71d671ba4d89c..afd3bee4c4ab8 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -784,7 +784,6 @@ public static class TestSearchResponse extends SearchResponse { 0L, ShardSearchFailure.EMPTY_ARRAY, Clusters.EMPTY, - null, null ); } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index 112befac30dd6..61aa05f703018 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -1059,7 +1059,6 @@ private static void resolveWithEmptySearchResponse(Tuple Date: Fri, 26 Sep 2025 18:05:21 +0200 Subject: [PATCH 03/30] iter --- .../org/elasticsearch/action/search/SearchPhaseController.java | 1 + .../src/main/java/org/elasticsearch/search/SearchService.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 8c39aa56e7545..f9f29f7e5c985 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -528,6 +528,7 @@ static ReducedQueryPhase reducedQueryPhase( } if (rangeTimestampFrom == null) { + //we simply take the first one: we should get the same value from all shards anyways rangeTimestampFrom = result.getRangeTimestampFrom(); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index c34259e936c73..1f7ad4b3d3f96 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1443,6 +1443,8 @@ private DefaultSearchContext createSearchContext( SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); if (context.getRangeTimestampFrom() != null) { + //range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place + //where we parse the date and set it to the context. We need to propagate it back from the clone into the original context searchContext.getSearchExecutionContext().setRangeTimestampFrom(context.getRangeTimestampFrom()); } assert searchContext.getSearchExecutionContext().isCacheable(); From 1aed53d428be51cdf78aa27bb994f84d17d27125 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 18:07:45 +0200 Subject: [PATCH 04/30] Update docs/changelog/135549.yaml --- docs/changelog/135549.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135549.yaml diff --git a/docs/changelog/135549.yaml b/docs/changelog/135549.yaml new file mode 100644 index 0000000000000..c3567161b7eaf --- /dev/null +++ b/docs/changelog/135549.yaml @@ -0,0 +1,5 @@ +pr: 135549 +summary: Add time range bucketing attribute to APM took time latency metrics +area: Search +type: enhancement +issues: [] From 6eb53a3dc2d51fb01182a694f7a2249c8d0f2444 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 26 Sep 2025 16:15:17 +0000 Subject: [PATCH 05/30] [CI] Auto commit changes from spotless --- .../elasticsearch/action/search/SearchPhaseController.java | 2 +- .../src/main/java/org/elasticsearch/search/SearchService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index f9f29f7e5c985..33fae1fe3f690 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -528,7 +528,7 @@ static ReducedQueryPhase reducedQueryPhase( } if (rangeTimestampFrom == null) { - //we simply take the first one: we should get the same value from all shards anyways + // we simply take the first one: we should get the same value from all shards anyways rangeTimestampFrom = result.getRangeTimestampFrom(); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 1f7ad4b3d3f96..64d9b4798ab5f 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1443,8 +1443,8 @@ private DefaultSearchContext createSearchContext( SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); if (context.getRangeTimestampFrom() != null) { - //range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place - //where we parse the date and set it to the context. We need to propagate it back from the clone into the original context + // range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place + // where we parse the date and set it to the context. We need to propagate it back from the clone into the original context searchContext.getSearchExecutionContext().setRangeTimestampFrom(context.getRangeTimestampFrom()); } assert searchContext.getSearchExecutionContext().isCacheable(); From 3967f72fda757c9d58212fe489efa2e7fa11d910 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 19:43:01 +0200 Subject: [PATCH 06/30] iter --- .../search/query/QueryPhase.java | 9 +-- .../SearchTookTimeTelemetryTests.java | 11 +++- .../search/query/QueryPhaseTests.java | 56 +++++++++---------- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index b07fe857e634a..3a33f902b641d 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -104,7 +104,8 @@ static void executeRank(SearchContext searchContext) throws QueryPhaseExecutionE queryPhaseRankShardContext.rankWindowSize() ) ) { - QueryPhase.addCollectorsAndSearch(rankSearchContext); + Long rangeTimestampFrom = searchContext.getSearchExecutionContext().getRangeTimestampFrom(); + QueryPhase.addCollectorsAndSearch(rankSearchContext, rangeTimestampFrom); QuerySearchResult rrfQuerySearchResult = rankSearchContext.queryResult(); rrfRankResults.add(rrfQuerySearchResult.topDocs().topDocs); serviceTimeEWMA += rrfQuerySearchResult.serviceTimeEWMA(); @@ -140,7 +141,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution // here to make sure it happens during the QUERY phase AggregationPhase.preProcess(searchContext); - addCollectorsAndSearch(searchContext); + addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getRangeTimestampFrom()); RescorePhase.execute(searchContext); SuggestPhase.execute(searchContext); @@ -153,11 +154,11 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution * In a package-private method so that it can be tested without having to * wire everything (mapperService, etc.) */ - static void addCollectorsAndSearch(SearchContext searchContext) throws QueryPhaseExecutionException { + static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimestampFrom) throws QueryPhaseExecutionException { final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); - queryResult.setRangeTimestampFrom(searchContext.getSearchExecutionContext().getRangeTimestampFrom()); + queryResult.setRangeTimestampFrom(rangeTimestampFrom); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index a6d59a903a457..7dc8bd8a74232 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -38,6 +38,7 @@ import org.junit.Before; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.Collection; @@ -56,7 +57,7 @@ public class SearchTookTimeTelemetryTests extends ESSingleNodeTestCase { private static final String indexName = "test_search_metrics2"; private static final String singleShardIndexName = "single_shard_test_search_metric"; - private static final LocalDateTime NOW = LocalDateTime.now(); + private static final LocalDateTime NOW = LocalDateTime.now(ZoneOffset.UTC); private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); @Override @@ -320,6 +321,14 @@ public void testCompoundRetriever() { assertEquals(2, measurements.size()); assertThat(measurements.getFirst().getLong(), Matchers.lessThan(searchResponse.getTook().millis())); assertEquals(searchResponse.getTook().millis(), measurements.getLast().getLong()); + for (Measurement measurement : measurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals("pit", attributes.get("pit_scroll")); + } } public void testMultiSearch() { diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 9f2fb0d91a1cc..3e7cd3d797ee0 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -142,7 +142,7 @@ private void countTestCase(Query query, IndexReader reader, boolean shouldCollec try (TestSearchContext context = createContext(searcher, query)) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); ContextIndexSearcher countSearcher = shouldCollectCount ? newContextSearcher(reader) : noCollectionContextSearcher(reader); assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value()); @@ -233,14 +233,14 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception { int numDocs = indexDocs(); try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) { // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold context.setSize(10); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -257,7 +257,7 @@ public void testPostFilterDisablesHitCountShortcut() throws Exception { // shortcutTotalHitCount is disabled for filter collectors, hence we collect until track_total_hits context.setSize(10); context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery())); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -269,7 +269,7 @@ public void testTerminateAfterWithFilter() throws Exception { context.terminateAfter(1); context.setSize(10); context.parsedPostFilter(new ParsedQuery(new TermQuery(new Term("foo", "bar")))); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -280,14 +280,14 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { int numDocs = indexDocs(); try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 10), new MatchAllDocsQuery())) { // shortcutTotalHitCount makes us not track total hits as part of the top docs collection, hence size is the threshold context.setSize(10); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(numDocs, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -296,7 +296,7 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { // the inner TotalHitCountCollector can shortcut context.setSize(0); context.minimumScore(100); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value()); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); } @@ -313,7 +313,7 @@ public void testMinScoreDisablesHitCountShortcut() throws Exception { public void testQueryCapturesThreadPoolStats() throws Exception { indexDocs(); try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); QuerySearchResult results = context.queryResult(); assertThat(results.serviceTimeEWMA(), greaterThanOrEqualTo(0L)); assertThat(results.nodeQueueSize(), greaterThanOrEqualTo(0)); @@ -336,7 +336,7 @@ public void testInOrderScrollOptimization() throws Exception { int size = randomIntBetween(2, 5); context.setSize(size); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); assertNull(context.queryResult().terminatedEarly()); @@ -344,7 +344,7 @@ public void testInOrderScrollOptimization() throws Exception { assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs)); context.setSearcher(earlyTerminationContextSearcher(reader, size)); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().getTotalHits().value(), equalTo((long) numDocs)); assertEquals(TotalHits.Relation.EQUAL_TO, context.queryResult().topDocs().topDocs.totalHits.relation()); @@ -363,7 +363,7 @@ public void testTerminateAfterSize0HitCountShortcut() throws Exception { try (TestSearchContext context = createContext(noCollectionContextSearcher(reader), new MatchAllDocsQuery())) { context.terminateAfter(1); context.setSize(0); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertFalse(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -476,7 +476,7 @@ public void testTerminateAfterWithHitsHitCountShortcut() throws Exception { context.terminateAfter(1); // default track_total_hits, size 1: terminate_after kicks in first context.setSize(1); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -534,7 +534,7 @@ public void testTerminateAfterWithHitsNoHitCountShortcut() throws Exception { try (TestSearchContext context = createContext(earlyTerminationContextSearcher(reader, 1), query)) { context.terminateAfter(1); context.setSize(1); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); @@ -600,7 +600,7 @@ public void testIndexSortingEarlyTermination() throws Exception { try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.totalHits.relation(), equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -612,7 +612,7 @@ public void testIndexSortingEarlyTermination() throws Exception { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); context.parsedPostFilter(new ParsedQuery(new MinDocQuery(1))); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo(numDocs - 1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -636,7 +636,7 @@ public void testIndexSortingEarlyTermination() throws Exception { context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); context.setSearcher(earlyTerminationContextSearcher(reader, 1)); context.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -646,7 +646,7 @@ public void testIndexSortingEarlyTermination() throws Exception { try (TestSearchContext context = createContext(newContextSearcher(reader), new MatchAllDocsQuery())) { context.setSize(1); context.sort(new SortAndFormats(sort, new DocValueFormat[] { DocValueFormat.RAW })); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -687,7 +687,7 @@ public void testIndexSortScrollOptimization() throws Exception { context.setSize(10); context.sort(searchSortAndFormat); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertNull(context.queryResult().terminatedEarly()); assertThat(context.terminateAfter(), equalTo(0)); @@ -695,7 +695,7 @@ public void testIndexSortScrollOptimization() throws Exception { int sizeMinus1 = context.queryResult().topDocs().topDocs.scoreDocs.length - 1; FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[sizeMinus1]; context.setSearcher(earlyTerminationContextSearcher(reader, 10)); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.terminateAfter(), equalTo(0)); @@ -823,7 +823,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -837,7 +837,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); final TopDocs topDocs = searchContext.queryResult().topDocs().topDocs; long firstResult = (long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]; @@ -850,7 +850,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsLongDate); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, true); } @@ -860,7 +860,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsDate); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, false); } @@ -870,7 +870,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.sort(formatsDateLong); searchContext.trackTotalHitsUpTo(10); searchContext.setSize(10); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertSortResults(searchContext.queryResult().topDocs().topDocs, numDocs, true); } @@ -881,7 +881,7 @@ public void testNumericSortOptimization() throws Exception { searchContext.trackTotalHitsUpTo(10); searchContext.from(5); searchContext.setSize(0); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints()); assertThat(searchContext.queryResult().topDocs().topDocs.scoreDocs, arrayWithSize(0)); assertThat(searchContext.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs - 1)); @@ -892,7 +892,7 @@ public void testNumericSortOptimization() throws Exception { try (TestSearchContext searchContext = createContext(newContextSearcher(reader), q)) { searchContext.sort(formatsLong); searchContext.setSize(0); - QueryPhase.addCollectorsAndSearch(searchContext); + QueryPhase.addCollectorsAndSearch(searchContext, null); } } @@ -992,7 +992,7 @@ public void testMinScore() throws Exception { context.setSize(1); context.trackTotalHitsUpTo(5); - QueryPhase.addCollectorsAndSearch(context); + QueryPhase.addCollectorsAndSearch(context, null); TotalHits totalHits = context.queryResult().topDocs().topDocs.totalHits; assertThat(totalHits.value(), greaterThanOrEqualTo(5L)); assertThat(totalHits.relation(), is(Relation.GREATER_THAN_OR_EQUAL_TO)); From 3eb28fa5449e2046221393bbd8e692e415b41f95 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 20:18:40 +0200 Subject: [PATCH 07/30] iter --- .../index/query/BoolQueryBuilder.java | 25 ++++- .../index/query/QueryRewriteContext.java | 20 +++- .../search/query/QueryPhase.java | 3 +- .../SearchTookTimeTelemetryTests.java | 101 +++++++++++++++++- 4 files changed, 138 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 1225a070a7c00..d309b5157f705 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -300,8 +300,13 @@ public String getWriteableName() { protected Query doToQuery(SearchExecutionContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); - addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); - addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); + try { + context.setTrackRangeTimestampFrom(false); + addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); + addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); + } finally { + context.setTrackRangeTimestampFrom(true); + } addBooleanClauses(context, booleanQueryBuilder, filterClauses, BooleanClause.Occur.FILTER); BooleanQuery booleanQuery = booleanQueryBuilder.build(); if (booleanQuery.clauses().isEmpty()) { @@ -348,9 +353,21 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return new MatchAllQueryBuilder().boost(boost()).queryName(queryName()); } changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must); - changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); + + try { + queryRewriteContext.setTrackRangeTimestampFrom(false); + changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); + } finally { + queryRewriteContext.setTrackRangeTimestampFrom(true); + } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); - changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); + try { + queryRewriteContext.setTrackRangeTimestampFrom(false); + changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); + } finally { + queryRewriteContext.setTrackRangeTimestampFrom(true); + } + // early termination when must clause is empty and optional clauses is returning MatchNoneQueryBuilder if (mustClauses.size() == 0 && filterClauses.size() == 0 diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index faca20858b586..7e7ff57c7bac8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -79,6 +79,7 @@ public class QueryRewriteContext { private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; private Long rangeTimestampFrom; + private boolean trackRangeTimestampFrom = true; public QueryRewriteContext( final XContentParserConfiguration parserConfiguration, @@ -532,10 +533,21 @@ public Long getRangeTimestampFrom() { * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. */ public void setRangeTimestampFrom(long rangeTimestampFrom) { - if (this.rangeTimestampFrom == null) { - this.rangeTimestampFrom = rangeTimestampFrom; - } else { - this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); + if (trackRangeTimestampFrom) { + if (this.rangeTimestampFrom == null) { + this.rangeTimestampFrom = rangeTimestampFrom; + } else { + this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); + } } } + + /** + * Enables or disables the tracking of the lower bound for time range filters against the @timestamp field, + * done via {@link #setRangeTimestampFrom(long)}. Tracking is enabled by default, and explicitly disabled to ensure + * we don't record the bound for range queries within should and must_not clauses. + */ + public void setTrackRangeTimestampFrom(boolean trackRangeTimestampFrom) { + this.trackRangeTimestampFrom = trackRangeTimestampFrom; + } } diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 3a33f902b641d..6b35a2e83acb9 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -104,8 +104,7 @@ static void executeRank(SearchContext searchContext) throws QueryPhaseExecutionE queryPhaseRankShardContext.rankWindowSize() ) ) { - Long rangeTimestampFrom = searchContext.getSearchExecutionContext().getRangeTimestampFrom(); - QueryPhase.addCollectorsAndSearch(rankSearchContext, rangeTimestampFrom); + QueryPhase.addCollectorsAndSearch(rankSearchContext, null); QuerySearchResult rrfQuerySearchResult = rankSearchContext.queryResult(); rrfRankResults.add(rrfQuerySearchResult.topDocs().topDocs); serviceTimeEWMA += rrfQuerySearchResult.serviceTimeEWMA(); diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 7dc8bd8a74232..8c28b8d3af43d 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -44,6 +44,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -58,7 +59,7 @@ public class SearchTookTimeTelemetryTests extends ESSingleNodeTestCase { private static final String indexName = "test_search_metrics2"; private static final String singleShardIndexName = "single_shard_test_search_metric"; private static final LocalDateTime NOW = LocalDateTime.now(ZoneOffset.UTC); - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT); @Override protected boolean resetNodeAfterTest() { @@ -503,6 +504,33 @@ public void testTimeRangeFilterAllResultsFilterOnEventIngested() { assertEquals(true, attributes.get("range_event_ingested")); } + public void testTimeRangeFilterAllResultsFilterOnEventIngestedAndTimestamp() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("event.ingested").from("2024-10-01")); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setPreFilterShardSize(1).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(6, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_event_ingested")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + } + public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(10)))); @@ -531,6 +559,77 @@ public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { assertEquals("15_minutes", attributes.get("timestamp_range_filter")); } + public void testMultipleTimeRangeFiltersQueryAndFetchRecentTimestamps() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + // we take the lowest of the two bounds + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(20)))); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(10)))); + // should and must_not get ignored + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(2)))); + boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(1)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) + .setQuery(boolQueryBuilder) + .addSort(new FieldSortBuilder("@timestamp")) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("@timestamp", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("1_hour", attributes.get("timestamp_range_filter")); + } + + public void testTimeRangeFilterAllResultsShouldClause() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from("2024-10-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "2"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + + public void testTimeRangeFilterOneResultMustNotClause() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from("2024-12-01")); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexName).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + assertSimpleQueryAttributes(measurement.attributes()); + } + private void resetMeter() { getTestTelemetryPlugin().resetMeter(); } From d9fdc5b42080f1fa43044e0adef99d16b1e1e214 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 26 Sep 2025 21:23:14 +0200 Subject: [PATCH 08/30] iter --- .../ShardSearchPhaseAPMMetricsTests.java | 22 ++++++-- .../search/query/QueryPhaseTests.java | 51 ++++++++++++++++++- .../search/query/QueryPhaseTimeoutTests.java | 45 +++++++++++++++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index 25fa7d1f36b41..ebae23382e93a 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -302,13 +302,29 @@ public void testTimeRangeFilterAllResults() { final List queryMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(QUERY_SEARCH_PHASE_METRIC); // the two docs are at most spread across two shards, other shards are empty and get filtered out assertThat(queryMeasurements.size(), Matchers.lessThanOrEqualTo(2)); - // no range info stored because we had no bounds after rewrite, basically a match_all - assertAttributes(queryMeasurements, false, false); + for (Measurement measurement : queryMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + //the range query was rewritten to one without bounds: we do track the from but we don't set the boolean flag + assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + } final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); // in this case, each shard queried has results to be fetched assertEquals(queryMeasurements.size(), fetchMeasurements.size()); // no range info stored because we had no bounds after rewrite, basically a match_all - assertAttributes(fetchMeasurements, false, false); + for (Measurement measurement : fetchMeasurements) { + Map attributes = measurement.attributes(); + assertEquals(4, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); + //no time range filter bucketing on the fetch phase, the query was rewritten to one without bounds + } } private void resetMeter() { diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 3e7cd3d797ee0..e0402b19dabdd 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -64,14 +64,26 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.lucene.queries.MinDocQuery; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -93,6 +105,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.elasticsearch.search.query.QueryPhaseCollectorManager.hasInfMaxScore; import static org.hamcrest.Matchers.anyOf; @@ -131,12 +144,46 @@ public void tearDown() throws Exception { } private TestSearchContext createContext(ContextIndexSearcher searcher, Query query) { - TestSearchContext context = new TestSearchContext(null, indexShard, searcher); + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, searcher); context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(query)); return context; } + private SearchExecutionContext createSearchExecutionContext() { + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + //final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + final long nowInMillis = randomNonNegativeLong(); + return new SearchExecutionContext( + 0, + 0, + indexSettings, + new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP), + (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + private void countTestCase(Query query, IndexReader reader, boolean shouldCollectSearch, boolean shouldCollectCount) throws Exception { ContextIndexSearcher searcher = shouldCollectSearch ? newContextSearcher(reader) : noCollectionContextSearcher(reader); try (TestSearchContext context = createContext(searcher, query)) { @@ -1047,7 +1094,7 @@ public T search(Query query, CollectorManager col } }; - try (SearchContext context = new TestSearchContext(null, indexShard, searcher) { + try (SearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, searcher) { @Override public Query buildFilteredQuery(Query query) { return query; diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index e19a85f4f2a0b..a036989b1310e 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -43,13 +43,22 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchShardTask; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.MapperMetrics; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ContextIndexSearcher; @@ -391,13 +400,47 @@ public long getRelativeTimeInMillis() { } private TestSearchContext createSearchContext(Query query, int size) throws IOException { - TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, newContextSearcher(reader)); context.setTask(new SearchShardTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(query)); context.setSize(size); return context; } + private SearchExecutionContext createSearchExecutionContext() { + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + //final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + final long nowInMillis = randomNonNegativeLong(); + return new SearchExecutionContext( + 0, + 0, + indexSettings, + new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP), + (ft, fdc) -> ft.fielddataBuilder(fdc).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + public void testSuggestOnlyWithTimeout() throws Exception { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().suggest(new SuggestBuilder()); try (SearchContext context = createSearchContextWithSuggestTimeout(searchSourceBuilder)) { From cc195d4ea8bb6f4780fa7f43012903e8c518796d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 26 Sep 2025 19:32:52 +0000 Subject: [PATCH 09/30] [CI] Auto commit changes from spotless --- .../TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java | 4 ++-- .../org/elasticsearch/search/query/QueryPhaseTests.java | 6 +----- .../elasticsearch/search/query/QueryPhaseTimeoutTests.java | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index ebae23382e93a..2fc432749d650 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -309,7 +309,7 @@ public void testTimeRangeFilterAllResults() { assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - //the range query was rewritten to one without bounds: we do track the from but we don't set the boolean flag + // the range query was rewritten to one without bounds: we do track the from but we don't set the boolean flag assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); } final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); @@ -323,7 +323,7 @@ public void testTimeRangeFilterAllResults() { assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - //no time range filter bucketing on the fetch phase, the query was rewritten to one without bounds + // no time range filter bucketing on the fetch phase, the query was rewritten to one without bounds } } diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index e0402b19dabdd..3c1b93e84690f 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -73,8 +73,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperMetrics; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.ParsedQuery; @@ -82,7 +80,6 @@ import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; -import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.lucene.queries.MinDocQuery; import org.elasticsearch.search.DocValueFormat; @@ -105,7 +102,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import static org.elasticsearch.search.query.QueryPhaseCollectorManager.hasInfMaxScore; import static org.hamcrest.Matchers.anyOf; @@ -158,7 +154,7 @@ private SearchExecutionContext createSearchExecutionContext() { .creationDate(System.currentTimeMillis()) .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - //final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + // final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); final long nowInMillis = randomNonNegativeLong(); return new SearchExecutionContext( 0, diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index a036989b1310e..f53e7cd96b547 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -415,7 +415,7 @@ private SearchExecutionContext createSearchExecutionContext() { .creationDate(System.currentTimeMillis()) .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - //final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); + // final SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); final long nowInMillis = randomNonNegativeLong(); return new SearchExecutionContext( 0, From b5f144d8721a297ab4b58f2625f4838f8df70b9a Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Sat, 27 Sep 2025 00:03:44 +0200 Subject: [PATCH 10/30] iter --- .../elasticsearch/search/query/QueryPhaseTimeoutTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java index f53e7cd96b547..e3eee4dea92f0 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTimeoutTests.java @@ -384,7 +384,7 @@ public long cost() { } private TestSearchContext createSearchContextWithTimeout(TimeoutQuery query, int size) throws IOException { - TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)) { + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, newContextSearcher(reader)) { @Override public long getRelativeTimeInMillis() { // this controls whether a timeout is raised or not. We abstract time away by pretending that the clock stops @@ -471,7 +471,7 @@ private TestSearchContext createSearchContextWithSuggestTimeout(SearchSourceBuil ContextIndexSearcher contextIndexSearcher = newContextSearcher(reader); SuggestionSearchContext suggestionSearchContext = new SuggestionSearchContext(); suggestionSearchContext.addSuggestion("suggestion", new TestSuggestionContext(new TestSuggester(contextIndexSearcher), null)); - TestSearchContext context = new TestSearchContext(null, indexShard, contextIndexSearcher) { + TestSearchContext context = new TestSearchContext(createSearchExecutionContext(), indexShard, contextIndexSearcher) { @Override public SuggestionSearchContext suggest() { return suggestionSearchContext; From 532eb96d322f9924639d8b12490ee1904d5d3193 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Sat, 27 Sep 2025 00:06:29 +0200 Subject: [PATCH 11/30] iter --- .../definitions/referable/timestamp_range_telemetry.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv index 4b012195cac35..0539697e70bda 100644 --- a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -1 +1 @@ -9177000 +9178000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 980b8e87c6329..43bd59db08ef4 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -extended_search_usage_telemetry,9177000 +timestamp_range_telemetry,9178000 From ff41cb0da6201608ac1d3154b7d36eb80832baea Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 20:06:40 +0200 Subject: [PATCH 12/30] Update server/src/main/java/org/elasticsearch/search/query/QueryPhase.java Co-authored-by: Andrei Dan --- .../main/java/org/elasticsearch/search/query/QueryPhase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 6b35a2e83acb9..85a8c55892df2 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -153,7 +153,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution * In a package-private method so that it can be tested without having to * wire everything (mapperService, etc.) */ - static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimestampFrom) throws QueryPhaseExecutionException { + static void addCollectorsAndSearch(SearchContext searchContext, @Nullable Long rangeTimestampFrom) throws QueryPhaseExecutionException { final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); From f98536101a17294f0fb903535740dd3108fbce83 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 20:06:57 +0200 Subject: [PATCH 13/30] Update server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java Co-authored-by: Andrei Dan --- .../java/org/elasticsearch/search/query/QuerySearchResult.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index d1e4281904f7f..a46a1fffe78ff 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -79,6 +79,7 @@ public final class QuerySearchResult extends SearchPhaseResult { private final SubscribableListener aggsContextReleased; + @Nullable private Long rangeTimestampFrom; public QuerySearchResult() { From 2ba5023f503a3bb96c7be9a5d4ef46e7ceda8a44 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 20:41:51 +0200 Subject: [PATCH 14/30] iter --- .../main/java/org/elasticsearch/search/query/QueryPhase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 85a8c55892df2..6b35a2e83acb9 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -153,7 +153,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution * In a package-private method so that it can be tested without having to * wire everything (mapperService, etc.) */ - static void addCollectorsAndSearch(SearchContext searchContext, @Nullable Long rangeTimestampFrom) throws QueryPhaseExecutionException { + static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimestampFrom) throws QueryPhaseExecutionException { final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); From d6637235acc72220aa49eb466ebb3e4bd92229a0 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 20:56:26 +0200 Subject: [PATCH 15/30] iter --- .../org/elasticsearch/search/query/QuerySearchResult.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index a46a1fffe78ff..0104b51534f44 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -458,7 +458,7 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del reduced = in.readBoolean(); } } - if (in.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { + if (in.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { rangeTimestampFrom = in.readOptionalLong(); } success = true; @@ -531,7 +531,7 @@ public void writeToNoId(StreamOutput out) throws IOException { if (versionSupportsBatchedExecution(out.getTransportVersion())) { out.writeBoolean(reduced); } - if (out.getTransportVersion().onOrAfter(TIMESTAMP_RANGE_TELEMETRY)) { + if (out.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { out.writeOptionalLong(rangeTimestampFrom); } } From 5eaf6db0b11048f7b4ba5fa64b7e0704a5f29f7c Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 21:08:54 +0200 Subject: [PATCH 16/30] iter --- .../org/elasticsearch/action/search/SearchPhaseController.java | 3 ++- .../java/org/elasticsearch/index/query/BoolQueryBuilder.java | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 33fae1fe3f690..1eb5f1e88e982 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -528,9 +528,10 @@ static ReducedQueryPhase reducedQueryPhase( } if (rangeTimestampFrom == null) { - // we simply take the first one: we should get the same value from all shards anyways + // we simply take the first one: we should get the same value from all shards anyway rangeTimestampFrom = result.getRangeTimestampFrom(); } + assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()); if (hasSuggest) { assert result.suggest() != null; diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index d309b5157f705..0167b18b6298b 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -301,6 +301,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); try { + //disable tracking of the @timestamp range for must_not and should clauses context.setTrackRangeTimestampFrom(false); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); @@ -355,6 +356,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must); try { + //disable tracking of the @timestamp range for must_not clauses queryRewriteContext.setTrackRangeTimestampFrom(false); changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); } finally { @@ -362,6 +364,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); try { + //disable tracking of the @timestamp range for should clauses queryRewriteContext.setTrackRangeTimestampFrom(false); changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); } finally { From 233db246a83b31e01856098a76877e130c88a5b4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 29 Sep 2025 19:17:00 +0000 Subject: [PATCH 17/30] [CI] Auto commit changes from spotless --- .../org/elasticsearch/index/query/BoolQueryBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 0167b18b6298b..9455b788bcc73 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -301,7 +301,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); try { - //disable tracking of the @timestamp range for must_not and should clauses + // disable tracking of the @timestamp range for must_not and should clauses context.setTrackRangeTimestampFrom(false); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); @@ -356,7 +356,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must); try { - //disable tracking of the @timestamp range for must_not clauses + // disable tracking of the @timestamp range for must_not clauses queryRewriteContext.setTrackRangeTimestampFrom(false); changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); } finally { @@ -364,7 +364,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); try { - //disable tracking of the @timestamp range for should clauses + // disable tracking of the @timestamp range for should clauses queryRewriteContext.setTrackRangeTimestampFrom(false); changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); } finally { From 53a80d0d0ea2742ca52ad8330f00e3b2056478b6 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 29 Sep 2025 22:10:47 +0200 Subject: [PATCH 18/30] iter --- .../org/elasticsearch/action/search/SearchPhaseController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 1eb5f1e88e982..05b71074c7414 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -531,7 +531,8 @@ static ReducedQueryPhase reducedQueryPhase( // we simply take the first one: we should get the same value from all shards anyway rangeTimestampFrom = result.getRangeTimestampFrom(); } - assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()); + assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) : + rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); if (hasSuggest) { assert result.suggest() != null; From 54a8a4f88e79ba2ace1b99a5f0b638335e22f49a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 29 Sep 2025 20:18:59 +0000 Subject: [PATCH 19/30] [CI] Auto commit changes from spotless --- .../elasticsearch/action/search/SearchPhaseController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 05b71074c7414..f87d2e26d2ac9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -531,8 +531,8 @@ static ReducedQueryPhase reducedQueryPhase( // we simply take the first one: we should get the same value from all shards anyway rangeTimestampFrom = result.getRangeTimestampFrom(); } - assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) : - rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); + assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) + : rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); if (hasSuggest) { assert result.suggest() != null; From 97cb335f9f290fd1edd7c3a92da900d74c952d0c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 29 Sep 2025 20:37:12 +0000 Subject: [PATCH 20/30] [CI] Update transport version definitions --- .../definitions/referable/timestamp_range_telemetry.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv index 51fcd22b7e793..c225f99e2e2c2 100644 --- a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -1 +1 @@ -9181000 +9182000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 9ebc18a7778ed..b35c51c1f75b8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -timestamp_range_telemetry,9181000 +timestamp_range_telemetry,9182000 From 3f47f9a903fa9731e57e95d9dfe3be8823ab2c4e Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 30 Sep 2025 21:44:53 +0200 Subject: [PATCH 21/30] iter --- .../action/search/SearchPhaseController.java | 5 +- .../index/query/QueryRewriteContext.java | 5 + .../SearchTookTimeTelemetryTests.java | 145 ++++++++++++++++-- 3 files changed, 138 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index f87d2e26d2ac9..d83143cf8f402 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -531,8 +531,9 @@ static ReducedQueryPhase reducedQueryPhase( // we simply take the first one: we should get the same value from all shards anyway rangeTimestampFrom = result.getRangeTimestampFrom(); } - assert rangeTimestampFrom == null || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) - : rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); + assert rangeTimestampFrom == null + || result.getRangeTimestampFrom() == null + || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) : rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); if (hasSuggest) { assert result.suggest() != null; diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 7e7ff57c7bac8..4fac94bcb4a5a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -534,9 +534,14 @@ public Long getRangeTimestampFrom() { */ public void setRangeTimestampFrom(long rangeTimestampFrom) { if (trackRangeTimestampFrom) { + // if we got a timestamp with nanoseconds precision, round it down to millis + if (rangeTimestampFrom > 1_000_000_000_000_000L) { + rangeTimestampFrom = rangeTimestampFrom / 1_000_000; + } if (this.rangeTimestampFrom == null) { this.rangeTimestampFrom = rangeTimestampFrom; } else { + // if there's more range filters on timestamp, we'll take the lowest of the lower bounds this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 8c28b8d3af43d..8abd1de3e9337 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -57,14 +57,11 @@ public class SearchTookTimeTelemetryTests extends ESSingleNodeTestCase { private static final String indexName = "test_search_metrics2"; + private static final String indexNameNanoPrecision = "nano_search_metrics2"; private static final String singleShardIndexName = "single_shard_test_search_metric"; private static final LocalDateTime NOW = LocalDateTime.now(ZoneOffset.UTC); - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT); - - @Override - protected boolean resetNodeAfterTest() { - return true; - } + private static final DateTimeFormatter FORMATTER_MILLIS = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT); + private static final DateTimeFormatter FORMATTER_NANOS = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn", Locale.ROOT); @Before public void setUpIndex() { @@ -93,11 +90,77 @@ public void setUpIndex() { ); ensureGreen(singleShardIndexName); prepareIndex(singleShardIndexName).setId("1") - .setSource("body", "foo", "@timestamp", NOW.minusMinutes(5).format(FORMATTER)) + .setSource("body", "foo", "@timestamp", NOW.minusMinutes(5).withSecond(randomIntBetween(0, 59)).format(FORMATTER_MILLIS)) .setRefreshPolicy(IMMEDIATE) .get(); prepareIndex(singleShardIndexName).setId("2") - .setSource("body", "foo", "@timestamp", NOW.minusMinutes(30).format(FORMATTER)) + .setSource("body", "foo", "@timestamp", NOW.minusMinutes(30).withSecond(randomIntBetween(0, 59)).format(FORMATTER_MILLIS)) + .setRefreshPolicy(IMMEDIATE) + .get(); + + createIndex( + indexNameNanoPrecision, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, num_primaries) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(), + "_doc", + "@timestamp", + "type=date_nanos" + ); + ensureGreen(indexNameNanoPrecision); + prepareIndex(indexNameNanoPrecision).setId("10") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(2).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("11") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(3).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("12") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(4).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("13") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(5).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("14") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(6).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) + .setRefreshPolicy(IMMEDIATE) + .get(); + prepareIndex(indexNameNanoPrecision).setId("15") + .setSource( + "body", + "foo", + "@timestamp", + NOW.minusMinutes(75).withNano(randomIntBetween(0, 1_000_000_000)).format(FORMATTER_NANOS) + ) .setRefreshPolicy(IMMEDIATE) .get(); } @@ -207,7 +270,7 @@ public void testOthersDottedIndexName() { SearchResponse searchResponse = client().prepareSearch("_all").setQuery(simpleQueryStringQuery("foo")).get(); try { assertNoFailures(searchResponse); - assertSearchHits(searchResponse, "1", "2", "1", "2"); + assertSearchHits(searchResponse, "1", "2", "1", "2", "10", "11", "12", "13", "14", "15"); } finally { searchResponse.decRef(); } @@ -320,7 +383,7 @@ public void testCompoundRetriever() { List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); // compound retriever does its own search as an async action, whose took time is recorded separately assertEquals(2, measurements.size()); - assertThat(measurements.getFirst().getLong(), Matchers.lessThan(searchResponse.getTook().millis())); + assertThat(measurements.getFirst().getLong(), Matchers.lessThanOrEqualTo(searchResponse.getTook().millis())); assertEquals(searchResponse.getTook().millis(), measurements.getLast().getLong()); for (Measurement measurement : measurements) { Map attributes = measurement.attributes(); @@ -533,7 +596,7 @@ public void testTimeRangeFilterAllResultsFilterOnEventIngestedAndTimestamp() { public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(10)))); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(10)))); boolQueryBuilder.must(simpleQueryStringQuery("foo")); SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) .setQuery(boolQueryBuilder) @@ -562,11 +625,11 @@ public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { public void testMultipleTimeRangeFiltersQueryAndFetchRecentTimestamps() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); // we take the lowest of the two bounds - boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(20)))); - boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(10)))); + boolQueryBuilder.must(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(10)))); // should and must_not get ignored - boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(2)))); - boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from(FORMATTER.format(NOW.minusMinutes(1)))); + boolQueryBuilder.should(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(2)))); + boolQueryBuilder.mustNot(new RangeQueryBuilder("@timestamp").from(FORMATTER_MILLIS.format(NOW.minusMinutes(1)))); boolQueryBuilder.must(simpleQueryStringQuery("foo")); SearchResponse searchResponse = client().prepareSearch(singleShardIndexName) .setQuery(boolQueryBuilder) @@ -630,6 +693,58 @@ public void testTimeRangeFilterOneResultMustNotClause() { assertSimpleQueryAttributes(measurement.attributes()); } + public void testTimeRangeFilterAllResultsNanoPrecision() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_NANOS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(indexNameNanoPrecision).setQuery(boolQueryBuilder).get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "10", "11", "12", "13", "14"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("1_hour", attributes.get("timestamp_range_filter")); + } + + public void testTimeRangeFilterAllResultsMixedPrecision() { + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.filter(new RangeQueryBuilder("@timestamp").from(FORMATTER_NANOS.format(NOW.minusMinutes(20)))); + boolQueryBuilder.must(simpleQueryStringQuery("foo")); + SearchResponse searchResponse = client().prepareSearch(singleShardIndexName, indexNameNanoPrecision) + .setQuery(boolQueryBuilder) + .get(); + try { + assertNoFailures(searchResponse); + assertSearchHits(searchResponse, "1", "10", "11", "12", "13", "14"); + } finally { + searchResponse.decRef(); + } + + List measurements = getTestTelemetryPlugin().getLongHistogramMeasurement(TOOK_DURATION_TOTAL_HISTOGRAM_NAME); + assertEquals(1, measurements.size()); + Measurement measurement = measurements.getFirst(); + assertEquals(searchResponse.getTook().millis(), measurement.getLong()); + Map attributes = measurement.attributes(); + assertEquals(5, attributes.size()); + assertEquals("user", attributes.get("target")); + assertEquals("hits_only", attributes.get("query_type")); + assertEquals("_score", attributes.get("sort")); + assertEquals(true, attributes.get("range_timestamp")); + assertEquals("1_hour", attributes.get("timestamp_range_filter")); + } + private void resetMeter() { getTestTelemetryPlugin().resetMeter(); } From e28898f762ef6e69650827f507a5bf7de08d02e2 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 30 Sep 2025 23:19:26 +0200 Subject: [PATCH 22/30] iter --- .../definitions/referable/timestamp_range_telemetry.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv index c225f99e2e2c2..02acf4c3b415f 100644 --- a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -1 +1 @@ -9182000 +9184000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index f575dcaf4efa8..af2054080a39c 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -esql_dense_vector_created_version,9183000 +timestamp_range_telemetry,9184000 From 415c1cfa1f43f5f6d754e288c7084ed0c7ab000b Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 14:14:24 +0200 Subject: [PATCH 23/30] iter --- .../definitions/referable/timestamp_range_telemetry.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.3.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv index 94a4d08acc2c7..c6a65950bd0ab 100644 --- a/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv +++ b/server/src/main/resources/transport/definitions/referable/timestamp_range_telemetry.csv @@ -1 +1 @@ -9185000 +9188000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index a1daf1f9747d4..fe8be2e1f55a8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -resolved_index_expressions,9187000 +timestamp_range_telemetry,9188000 From 5d9621ee95de798689bb008e6f350f658a42ee96 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 14:39:46 +0200 Subject: [PATCH 24/30] rename --- .../action/search/RankFeaturePhase.java | 2 +- .../action/search/SearchPhaseController.java | 18 ++++++------ .../SearchRequestAttributesExtractor.java | 14 +++++----- .../action/search/SearchResponse.java | 8 +++--- .../action/search/SearchResponseSections.java | 6 ++-- .../action/search/TransportSearchAction.java | 2 +- .../index/mapper/DateFieldMapper.java | 4 +-- .../index/query/BoolQueryBuilder.java | 12 ++++---- .../index/query/QueryRewriteContext.java | 28 +++++++++---------- .../stats/ShardSearchPhaseAPMMetrics.java | 12 ++++---- .../action/search/SearchResponseMetrics.java | 4 +-- .../elasticsearch/search/SearchService.java | 4 +-- .../search/query/QueryPhase.java | 4 +-- .../search/query/QuerySearchResult.java | 14 +++++----- 14 files changed, 66 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index c0f1566b2b281..355dc38c007a0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -252,7 +252,7 @@ private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults( reducedQueryPhase.size(), reducedQueryPhase.from(), reducedQueryPhase.isEmptyResult(), - reducedQueryPhase.rangeTimestampFrom() + reducedQueryPhase.rangeTimestampFromMillis() ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index d83143cf8f402..10cb66a909556 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -517,7 +517,7 @@ static ReducedQueryPhase reducedQueryPhase( : Collections.emptyMap(); int from = 0; int size = 0; - Long rangeTimestampFrom = null; + Long rangeTimestampFromMillis = null; DocValueFormat[] sortValueFormats = null; for (QuerySearchResult result : nonNullResults) { from = result.from(); @@ -527,13 +527,13 @@ static ReducedQueryPhase reducedQueryPhase( sortValueFormats = result.sortValueFormats(); } - if (rangeTimestampFrom == null) { + if (rangeTimestampFromMillis == null) { // we simply take the first one: we should get the same value from all shards anyway - rangeTimestampFrom = result.getRangeTimestampFrom(); + rangeTimestampFromMillis = result.getRangeTimestampFromMillis(); } - assert rangeTimestampFrom == null - || result.getRangeTimestampFrom() == null - || rangeTimestampFrom.equals(result.getRangeTimestampFrom()) : rangeTimestampFrom + " != " + result.getRangeTimestampFrom(); + assert rangeTimestampFromMillis == null + || result.getRangeTimestampFromMillis() == null + || rangeTimestampFromMillis.equals(result.getRangeTimestampFromMillis()) : rangeTimestampFromMillis + " != " + result.getRangeTimestampFromMillis(); if (hasSuggest) { assert result.suggest() != null; @@ -590,7 +590,7 @@ static ReducedQueryPhase reducedQueryPhase( size, from, false, - rangeTimestampFrom + rangeTimestampFromMillis ); } @@ -674,7 +674,7 @@ public record ReducedQueryPhase( int from, // true iff the query phase had no results. Otherwise false boolean isEmptyResult, - Long rangeTimestampFrom + Long rangeTimestampFromMillis ) { public ReducedQueryPhase { @@ -696,7 +696,7 @@ public SearchResponseSections buildResponse(SearchHits hits, Collection extractAttributes(SearchRequest searchRequest, /** * Introspects the provided shard search request and extracts metadata from it about some of its characteristics. */ - public static Map extractAttributes(ShardSearchRequest shardSearchRequest, Long rangeTimestampFrom, long nowInMillis) { + public static Map extractAttributes(ShardSearchRequest shardSearchRequest, Long rangeTimestampFromMillis, long nowInMillis) { Map attributes = extractAttributes( shardSearchRequest.source(), shardSearchRequest.scroll(), - rangeTimestampFrom, + rangeTimestampFromMillis, nowInMillis, shardSearchRequest.shardId().getIndexName() ); @@ -69,7 +69,7 @@ public static Map extractAttributes(ShardSearchRequest shardSear private static Map extractAttributes( SearchSourceBuilder searchSourceBuilder, TimeValue scroll, - Long rangeTimestampFrom, + Long rangeTimestampFromMillis, long nowInMillis, String... localIndices ) { @@ -108,8 +108,8 @@ private static Map extractAttributes( final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; String timestampRangeFilter = null; - if (rangeTimestampFrom != null) { - timestampRangeFilter = introspectTimeRange(rangeTimestampFrom, nowInMillis); + if (rangeTimestampFromMillis != null) { + timestampRangeFilter = introspectTimeRange(rangeTimestampFromMillis, nowInMillis); } return buildAttributesMap( target, @@ -350,9 +350,9 @@ public static void addTimeRangeAttribute(Long timeRangeFrom, long nowInMillis, M } } - static String introspectTimeRange(long timeRangeFrom, long nowInMillis) { + static String introspectTimeRange(long timeRangeFromMillis, long nowInMillis) { for (TimeRangeBucket value : TimeRangeBucket.values()) { - if (timeRangeFrom >= nowInMillis - value.millis) { + if (timeRangeFromMillis >= nowInMillis - value.millis) { return value.bucketName; } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index ebb16f57aa576..c31c1d881bcb2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -88,7 +88,7 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO private final Clusters clusters; private final long tookInMillis; // only used for telemetry purposes on the coordinating node, where the search response gets created - private transient Long rangeTimestampFrom; + private transient Long rangeTimestampFromMillis; private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted()); @@ -189,7 +189,7 @@ public SearchResponse( clusters, pointInTimeId ); - this.rangeTimestampFrom = searchResponseSections.rangeTimestampFrom; + this.rangeTimestampFromMillis = searchResponseSections.rangeTimestampFromMillis; } public SearchResponse( @@ -467,8 +467,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBytesReference(pointInTimeId); } - public Long getRangeTimestampFrom() { - return rangeTimestampFrom; + public Long getRangeTimestampFromMillis() { + return rangeTimestampFromMillis; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java index 07bf7d99d6277..afc23e336f905 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -52,7 +52,7 @@ public class SearchResponseSections implements Releasable { protected final boolean timedOut; protected final Boolean terminatedEarly; protected final int numReducePhases; - protected final Long rangeTimestampFrom; + protected final Long rangeTimestampFromMillis; public SearchResponseSections( SearchHits hits, @@ -62,7 +62,7 @@ public SearchResponseSections( Boolean terminatedEarly, SearchProfileResults profileResults, int numReducePhases, - Long rangeTimestampFrom + Long rangeTimestampFromMillis ) { this.hits = hits; this.aggregations = aggregations; @@ -71,7 +71,7 @@ public SearchResponseSections( this.timedOut = timedOut; this.terminatedEarly = terminatedEarly; this.numReducePhases = numReducePhases; - this.rangeTimestampFrom = rangeTimestampFrom; + this.rangeTimestampFromMillis = rangeTimestampFromMillis; } public final SearchHits hits() { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 4447986e6b315..68ce31e8bda7d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -2094,7 +2094,7 @@ public void onResponse(SearchResponse searchResponse) { try { searchResponseMetrics.recordTookTime( searchResponse.getTookInMillis(), - searchResponse.getRangeTimestampFrom(), + searchResponse.getRangeTimestampFromMillis(), nowInMillis, searchRequestAttributes ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index b472cfcdea9f2..2ec452f63fd1b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -805,7 +805,7 @@ public static Query dateRangeQuery( ++l; } if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { - context.setRangeTimestampFrom(l); + context.setRangeTimestampFromMillis(l); } } if (upperTerm == null) { @@ -997,7 +997,7 @@ public Relation isFieldWithinQuery( ++fromInclusive; } if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { - context.setRangeTimestampFrom(fromInclusive); + context.setRangeTimestampFromMillis(fromInclusive); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 9455b788bcc73..9b0b37d5bfc47 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -302,11 +302,11 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); try { // disable tracking of the @timestamp range for must_not and should clauses - context.setTrackRangeTimestampFrom(false); + context.setTrackRangeTimestampFromMillis(false); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); } finally { - context.setTrackRangeTimestampFrom(true); + context.setTrackRangeTimestampFromMillis(true); } addBooleanClauses(context, booleanQueryBuilder, filterClauses, BooleanClause.Occur.FILTER); BooleanQuery booleanQuery = booleanQueryBuilder.build(); @@ -357,18 +357,18 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws try { // disable tracking of the @timestamp range for must_not clauses - queryRewriteContext.setTrackRangeTimestampFrom(false); + queryRewriteContext.setTrackRangeTimestampFromMillis(false); changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); } finally { - queryRewriteContext.setTrackRangeTimestampFrom(true); + queryRewriteContext.setTrackRangeTimestampFromMillis(true); } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); try { // disable tracking of the @timestamp range for should clauses - queryRewriteContext.setTrackRangeTimestampFrom(false); + queryRewriteContext.setTrackRangeTimestampFromMillis(false); changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); } finally { - queryRewriteContext.setTrackRangeTimestampFrom(true); + queryRewriteContext.setTrackRangeTimestampFromMillis(true); } // early termination when must clause is empty and optional clauses is returning MatchNoneQueryBuilder diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index ca29647a1b554..0218d1b0da06d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -77,8 +77,8 @@ public class QueryRewriteContext { private QueryRewriteInterceptor queryRewriteInterceptor; private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; - private Long rangeTimestampFrom; - private boolean trackRangeTimestampFrom = true; + private Long rangeTimestampFromMillis; + private boolean trackRangeTimestampFromMillis = true; public QueryRewriteContext( final XContentParserConfiguration parserConfiguration, @@ -520,34 +520,34 @@ public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInter /** * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query */ - public Long getRangeTimestampFrom() { - return rangeTimestampFrom; + public Long getRangeTimestampFromMillis() { + return rangeTimestampFromMillis; } /** * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. */ - public void setRangeTimestampFrom(long rangeTimestampFrom) { - if (trackRangeTimestampFrom) { + public void setRangeTimestampFromMillis(long rangeTimestampFromMillis) { + if (trackRangeTimestampFromMillis) { // if we got a timestamp with nanoseconds precision, round it down to millis - if (rangeTimestampFrom > 1_000_000_000_000_000L) { - rangeTimestampFrom = rangeTimestampFrom / 1_000_000; + if (rangeTimestampFromMillis > 1_000_000_000_000_000L) { + rangeTimestampFromMillis = rangeTimestampFromMillis / 1_000_000; } - if (this.rangeTimestampFrom == null) { - this.rangeTimestampFrom = rangeTimestampFrom; + if (this.rangeTimestampFromMillis == null) { + this.rangeTimestampFromMillis = rangeTimestampFromMillis; } else { // if there's more range filters on timestamp, we'll take the lowest of the lower bounds - this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom); + this.rangeTimestampFromMillis = Math.min(rangeTimestampFromMillis, this.rangeTimestampFromMillis); } } } /** * Enables or disables the tracking of the lower bound for time range filters against the @timestamp field, - * done via {@link #setRangeTimestampFrom(long)}. Tracking is enabled by default, and explicitly disabled to ensure + * done via {@link #setRangeTimestampFromMillis(long)}. Tracking is enabled by default, and explicitly disabled to ensure * we don't record the bound for range queries within should and must_not clauses. */ - public void setTrackRangeTimestampFrom(boolean trackRangeTimestampFrom) { - this.trackRangeTimestampFrom = trackRangeTimestampFrom; + public void setTrackRangeTimestampFromMillis(boolean trackRangeTimestampFromMillis) { + this.trackRangeTimestampFromMillis = trackRangeTimestampFromMillis; } } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index fc823f4c51f7c..e37fea974fea9 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -56,15 +56,15 @@ public void onDfsPhase(SearchContext searchContext, long tookInNanos) { @Override public void onQueryPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); - recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); + Long rangeTimestampFromMillis = searchExecutionContext.getRangeTimestampFromMillis(); + recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFromMillis); } @Override public void onFetchPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); - recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); + Long rangeTimestampFromMillis = searchExecutionContext.getRangeTimestampFromMillis(); + recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFromMillis); } private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) { @@ -75,11 +75,11 @@ private static void recordPhaseLatency( LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request, - Long rangeTimestampFrom + Long rangeTimestampFromMillis ) { Map attributes = SearchRequestAttributesExtractor.extractAttributes( request, - rangeTimestampFrom, + rangeTimestampFromMillis, request.nowInMillis() ); histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 50702c0ea89b5..749bdfcd070df 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -73,8 +73,8 @@ public long recordTookTimeForSearchScroll(long tookTime) { return tookTime; } - public long recordTookTime(long tookTime, Long rangeTimestampFrom, long nowInMillis, Map attributes) { - SearchRequestAttributesExtractor.addTimeRangeAttribute(rangeTimestampFrom, nowInMillis, attributes); + public long recordTookTime(long tookTime, Long rangeTimestampFromMillis, long nowInMillis, Map attributes) { + SearchRequestAttributesExtractor.addTimeRangeAttribute(rangeTimestampFromMillis, nowInMillis, attributes); tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 64d9b4798ab5f..2954b3cd0a4bb 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1442,10 +1442,10 @@ private DefaultSearchContext createSearchContext( // during rewrite and normalized / evaluate templates etc. SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); - if (context.getRangeTimestampFrom() != null) { + if (context.getRangeTimestampFromMillis() != null) { // range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place // where we parse the date and set it to the context. We need to propagate it back from the clone into the original context - searchContext.getSearchExecutionContext().setRangeTimestampFrom(context.getRangeTimestampFrom()); + searchContext.getSearchExecutionContext().setRangeTimestampFromMillis(context.getRangeTimestampFromMillis()); } assert searchContext.getSearchExecutionContext().isCacheable(); success = true; diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 6b35a2e83acb9..fcf2c4f254cc0 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -140,7 +140,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution // here to make sure it happens during the QUERY phase AggregationPhase.preProcess(searchContext); - addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getRangeTimestampFrom()); + addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getRangeTimestampFromMillis()); RescorePhase.execute(searchContext); SuggestPhase.execute(searchContext); @@ -157,7 +157,7 @@ static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimest final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); - queryResult.setRangeTimestampFrom(rangeTimestampFrom); + queryResult.setRangeTimestampFromMillis(rangeTimestampFrom); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index 0104b51534f44..bd0210585ce52 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -80,7 +80,7 @@ public final class QuerySearchResult extends SearchPhaseResult { private final SubscribableListener aggsContextReleased; @Nullable - private Long rangeTimestampFrom; + private Long rangeTimestampFromMillis; public QuerySearchResult() { this(false); @@ -459,7 +459,7 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del } } if (in.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { - rangeTimestampFrom = in.readOptionalLong(); + rangeTimestampFromMillis = in.readOptionalLong(); } success = true; } finally { @@ -532,7 +532,7 @@ public void writeToNoId(StreamOutput out) throws IOException { out.writeBoolean(reduced); } if (out.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { - out.writeOptionalLong(rangeTimestampFrom); + out.writeOptionalLong(rangeTimestampFromMillis); } } @@ -587,11 +587,11 @@ private static boolean versionSupportsBatchedExecution(TransportVersion transpor || transportVersion.isPatchFrom(TransportVersions.BATCHED_QUERY_PHASE_VERSION_BACKPORT_8_X); } - public Long getRangeTimestampFrom() { - return rangeTimestampFrom; + public Long getRangeTimestampFromMillis() { + return rangeTimestampFromMillis; } - public void setRangeTimestampFrom(Long rangeTimestampFrom) { - this.rangeTimestampFrom = rangeTimestampFrom; + public void setRangeTimestampFromMillis(Long rangeTimestampFromMillis) { + this.rangeTimestampFromMillis = rangeTimestampFromMillis; } } From dfb60c629502efd58e01ffc3812e23d407d3a5fc Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 6 Oct 2025 14:40:48 +0000 Subject: [PATCH 25/30] [CI] Auto commit changes from spotless --- .../elasticsearch/action/search/SearchPhaseController.java | 3 ++- .../action/search/SearchRequestAttributesExtractor.java | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 10cb66a909556..5caa59aec648a 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -533,7 +533,8 @@ static ReducedQueryPhase reducedQueryPhase( } assert rangeTimestampFromMillis == null || result.getRangeTimestampFromMillis() == null - || rangeTimestampFromMillis.equals(result.getRangeTimestampFromMillis()) : rangeTimestampFromMillis + " != " + result.getRangeTimestampFromMillis(); + || rangeTimestampFromMillis.equals(result.getRangeTimestampFromMillis()) + : rangeTimestampFromMillis + " != " + result.getRangeTimestampFromMillis(); if (hasSuggest) { assert result.suggest() != null; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java index 2d00dea23e38d..b396f61feab8c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java @@ -53,7 +53,11 @@ public static Map extractAttributes(SearchRequest searchRequest, /** * Introspects the provided shard search request and extracts metadata from it about some of its characteristics. */ - public static Map extractAttributes(ShardSearchRequest shardSearchRequest, Long rangeTimestampFromMillis, long nowInMillis) { + public static Map extractAttributes( + ShardSearchRequest shardSearchRequest, + Long rangeTimestampFromMillis, + long nowInMillis + ) { Map attributes = extractAttributes( shardSearchRequest.source(), shardSearchRequest.scroll(), From 86d574619a21f0a02390d97c2d876ee76d0630f8 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 20:52:45 +0200 Subject: [PATCH 26/30] iter --- .../action/search/RankFeaturePhase.java | 2 +- .../action/search/SearchPhaseController.java | 21 ++++---- .../SearchRequestAttributesExtractor.java | 47 +++++++++------- .../action/search/SearchResponse.java | 8 +-- .../action/search/SearchResponseSections.java | 6 +-- .../action/search/TransportSearchAction.java | 2 +- .../index/mapper/DateFieldMapper.java | 11 ++-- .../index/query/BoolQueryBuilder.java | 12 ++--- .../index/query/QueryRewriteContext.java | 54 ++++++++++++------- .../stats/ShardSearchPhaseAPMMetrics.java | 12 ++--- .../action/search/SearchResponseMetrics.java | 4 +- .../elasticsearch/search/SearchService.java | 4 +- .../search/query/QueryPhase.java | 4 +- .../search/query/QuerySearchResult.java | 14 ++--- ...SearchRequestAttributesExtractorTests.java | 15 +++--- .../SearchTookTimeTelemetryTests.java | 34 ++++++------ .../ShardSearchPhaseAPMMetricsTests.java | 11 ++-- .../AsyncSearchTookTimeTelemetryTests.java | 4 +- 18 files changed, 143 insertions(+), 122 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 355dc38c007a0..07b183629fcb5 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -252,7 +252,7 @@ private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults( reducedQueryPhase.size(), reducedQueryPhase.from(), reducedQueryPhase.isEmptyResult(), - reducedQueryPhase.rangeTimestampFromMillis() + reducedQueryPhase.timeRangeFilterFromMillis() ); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 5caa59aec648a..d39dd90dbc780 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -517,7 +517,7 @@ static ReducedQueryPhase reducedQueryPhase( : Collections.emptyMap(); int from = 0; int size = 0; - Long rangeTimestampFromMillis = null; + Long timeRangeFilterFromMillis = null; DocValueFormat[] sortValueFormats = null; for (QuerySearchResult result : nonNullResults) { from = result.from(); @@ -527,14 +527,15 @@ static ReducedQueryPhase reducedQueryPhase( sortValueFormats = result.sortValueFormats(); } - if (rangeTimestampFromMillis == null) { + if (timeRangeFilterFromMillis == null) { // we simply take the first one: we should get the same value from all shards anyway - rangeTimestampFromMillis = result.getRangeTimestampFromMillis(); + timeRangeFilterFromMillis = result.getTimeRangeFilterFromMillis(); } - assert rangeTimestampFromMillis == null - || result.getRangeTimestampFromMillis() == null - || rangeTimestampFromMillis.equals(result.getRangeTimestampFromMillis()) - : rangeTimestampFromMillis + " != " + result.getRangeTimestampFromMillis(); + + assert timeRangeFilterFromMillis == null + || result.getTimeRangeFilterFromMillis() == null + || timeRangeFilterFromMillis.equals(result.getTimeRangeFilterFromMillis()) + : timeRangeFilterFromMillis + " != " + result.getTimeRangeFilterFromMillis(); if (hasSuggest) { assert result.suggest() != null; @@ -591,7 +592,7 @@ static ReducedQueryPhase reducedQueryPhase( size, from, false, - rangeTimestampFromMillis + timeRangeFilterFromMillis ); } @@ -675,7 +676,7 @@ public record ReducedQueryPhase( int from, // true iff the query phase had no results. Otherwise false boolean isEmptyResult, - Long rangeTimestampFromMillis + Long timeRangeFilterFromMillis ) { public ReducedQueryPhase { @@ -697,7 +698,7 @@ public SearchResponseSections buildResponse(SearchHits hits, Collection extractAttributes(SearchRequest searchRequest, */ public static Map extractAttributes( ShardSearchRequest shardSearchRequest, - Long rangeTimestampFromMillis, + Long timeRangeFilterFromMillis, long nowInMillis ) { Map attributes = extractAttributes( shardSearchRequest.source(), shardSearchRequest.scroll(), - rangeTimestampFromMillis, + timeRangeFilterFromMillis, nowInMillis, shardSearchRequest.shardId().getIndexName() ); @@ -73,7 +75,7 @@ public static Map extractAttributes( private static Map extractAttributes( SearchSourceBuilder searchSourceBuilder, TimeValue scroll, - Long rangeTimestampFromMillis, + Long timeRangeFilterFromMillis, long nowInMillis, String... localIndices ) { @@ -111,9 +113,9 @@ private static Map extractAttributes( } final boolean hasKnn = searchSourceBuilder.knnSearch().isEmpty() == false || queryMetadataBuilder.knnQuery; - String timestampRangeFilter = null; - if (rangeTimestampFromMillis != null) { - timestampRangeFilter = introspectTimeRange(rangeTimestampFromMillis, nowInMillis); + String timeRangeFilterFrom = null; + if (timeRangeFilterFromMillis != null) { + timeRangeFilterFrom = introspectTimeRange(timeRangeFilterFromMillis, nowInMillis); } return buildAttributesMap( target, @@ -123,7 +125,7 @@ private static Map extractAttributes( queryMetadataBuilder.rangeOnTimestamp, queryMetadataBuilder.rangeOnEventIngested, pitOrScroll, - timestampRangeFilter + timeRangeFilterFrom ); } @@ -135,7 +137,7 @@ private static Map buildAttributesMap( boolean rangeOnTimestamp, boolean rangeOnEventIngested, String pitOrScroll, - String timestampRangeFilter + String timeRangeFilterFrom ) { Map attributes = new HashMap<>(5, 1.0f); attributes.put(TARGET_ATTRIBUTE, target); @@ -147,14 +149,18 @@ private static Map buildAttributesMap( if (knn) { attributes.put(KNN_ATTRIBUTE, knn); } - if (rangeOnTimestamp) { - attributes.put(RANGE_TIMESTAMP_ATTRIBUTE, rangeOnTimestamp); + if (rangeOnTimestamp && rangeOnEventIngested) { + attributes.put( + TIME_RANGE_FILTER_FIELD_ATTRIBUTE, + DataStream.TIMESTAMP_FIELD_NAME + "_AND_" + IndexMetadata.EVENT_INGESTED_FIELD_NAME + ); + } else if (rangeOnEventIngested) { + attributes.put(TIME_RANGE_FILTER_FIELD_ATTRIBUTE, IndexMetadata.EVENT_INGESTED_FIELD_NAME); + } else if (rangeOnTimestamp) { + attributes.put(TIME_RANGE_FILTER_FIELD_ATTRIBUTE, DataStream.TIMESTAMP_FIELD_NAME); } - if (rangeOnEventIngested) { - attributes.put(RANGE_EVENT_INGESTED_ATTRIBUTE, rangeOnEventIngested); - } - if (timestampRangeFilter != null) { - attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter); + if (timeRangeFilterFrom != null) { + attributes.put(TIME_RANGE_FILTER_FROM_ATTRIBUTE, timeRangeFilterFrom); } return attributes; } @@ -170,9 +176,8 @@ private static final class QueryMetadataBuilder { static final String QUERY_TYPE_ATTRIBUTE = "query_type"; static final String PIT_SCROLL_ATTRIBUTE = "pit_scroll"; static final String KNN_ATTRIBUTE = "knn"; - static final String RANGE_TIMESTAMP_ATTRIBUTE = "range_timestamp"; - static final String RANGE_EVENT_INGESTED_ATTRIBUTE = "range_event_ingested"; - static final String TIMESTAMP_RANGE_FILTER_ATTRIBUTE = "timestamp_range_filter"; + static final String TIME_RANGE_FILTER_FIELD_ATTRIBUTE = "time_range_filter_field"; + static final String TIME_RANGE_FILTER_FROM_ATTRIBUTE = "time_range_filter_from"; private static final String TARGET_KIBANA = ".kibana"; private static final String TARGET_ML = ".ml"; @@ -307,6 +312,10 @@ private static void introspectQueryBuilder(QueryBuilder queryBuilder, QueryMetad introspectQueryBuilder(nested.query(), queryMetadataBuilder, ++level); break; case RangeQueryBuilder range: + // Note that the outcome of this switch differs depending on whether it is executed on the coord node, or data node. + // Data nodes perform query rewrite on each shard. That means that a query that reports a certain time range filter at the + // coordinator, may not report the same for all the shards it targets, but rather only for those that do end up executing + // a true range query at the shard level. switch (range.fieldName()) { // don't track unbounded ranges, they translate to either match_none if the field does not exist // or match_all if the field is mapped @@ -350,7 +359,7 @@ private enum TimeRangeBucket { public static void addTimeRangeAttribute(Long timeRangeFrom, long nowInMillis, Map attributes) { if (timeRangeFrom != null) { String timestampRangeFilter = introspectTimeRange(timeRangeFrom, nowInMillis); - attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter); + attributes.put(TIME_RANGE_FILTER_FROM_ATTRIBUTE, timestampRangeFilter); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index c31c1d881bcb2..3b67d0b5ac160 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -88,7 +88,7 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO private final Clusters clusters; private final long tookInMillis; // only used for telemetry purposes on the coordinating node, where the search response gets created - private transient Long rangeTimestampFromMillis; + private transient Long timeRangeFilterFromMillis; private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted()); @@ -189,7 +189,7 @@ public SearchResponse( clusters, pointInTimeId ); - this.rangeTimestampFromMillis = searchResponseSections.rangeTimestampFromMillis; + this.timeRangeFilterFromMillis = searchResponseSections.timeRangeFilterFromMillis; } public SearchResponse( @@ -467,8 +467,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalBytesReference(pointInTimeId); } - public Long getRangeTimestampFromMillis() { - return rangeTimestampFromMillis; + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java index afc23e336f905..c13578d7aeb9b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java @@ -52,7 +52,7 @@ public class SearchResponseSections implements Releasable { protected final boolean timedOut; protected final Boolean terminatedEarly; protected final int numReducePhases; - protected final Long rangeTimestampFromMillis; + protected final Long timeRangeFilterFromMillis; public SearchResponseSections( SearchHits hits, @@ -62,7 +62,7 @@ public SearchResponseSections( Boolean terminatedEarly, SearchProfileResults profileResults, int numReducePhases, - Long rangeTimestampFromMillis + Long timeRangeFilterFromMillis ) { this.hits = hits; this.aggregations = aggregations; @@ -71,7 +71,7 @@ public SearchResponseSections( this.timedOut = timedOut; this.terminatedEarly = terminatedEarly; this.numReducePhases = numReducePhases; - this.rangeTimestampFromMillis = rangeTimestampFromMillis; + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; } public final SearchHits hits() { diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 68ce31e8bda7d..16389f6137f81 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -2094,7 +2094,7 @@ public void onResponse(SearchResponse searchResponse) { try { searchResponseMetrics.recordTookTime( searchResponse.getTookInMillis(), - searchResponse.getRangeTimestampFromMillis(), + searchResponse.getTimeRangeFilterFromMillis(), nowInMillis, searchRequestAttributes ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2ec452f63fd1b..030c202b08080 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -804,9 +803,7 @@ public static Query dateRangeQuery( if (includeLower == false) { ++l; } - if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { - context.setRangeTimestampFromMillis(l); - } + context.setTimeRangeFilterFromMillis(fieldName, l, resolution); } if (upperTerm == null) { u = Long.MAX_VALUE; @@ -996,9 +993,9 @@ public Relation isFieldWithinQuery( } ++fromInclusive; } - if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) { - context.setRangeTimestampFromMillis(fromInclusive); - } + // we set the time range filter from during rewrite, because this may be the only time we ever parse it, + // in case the shard if filtered out and does not run the query phase or all its docs are within the bounds. + context.setTimeRangeFilterFromMillis(fieldName, fromInclusive, resolution); } long toInclusive = Long.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 9b0b37d5bfc47..5944fc3d8df7a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -302,11 +302,11 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST); try { // disable tracking of the @timestamp range for must_not and should clauses - context.setTrackRangeTimestampFromMillis(false); + context.setTrackTimeRangeFilterFrom(false); addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT); addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD); } finally { - context.setTrackRangeTimestampFromMillis(true); + context.setTrackTimeRangeFilterFrom(true); } addBooleanClauses(context, booleanQueryBuilder, filterClauses, BooleanClause.Occur.FILTER); BooleanQuery booleanQuery = booleanQueryBuilder.build(); @@ -357,18 +357,18 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws try { // disable tracking of the @timestamp range for must_not clauses - queryRewriteContext.setTrackRangeTimestampFromMillis(false); + queryRewriteContext.setTrackTimeRangeFilterFrom(false); changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot); } finally { - queryRewriteContext.setTrackRangeTimestampFromMillis(true); + queryRewriteContext.setTrackTimeRangeFilterFrom(true); } changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter); try { // disable tracking of the @timestamp range for should clauses - queryRewriteContext.setTrackRangeTimestampFromMillis(false); + queryRewriteContext.setTrackTimeRangeFilterFrom(false); changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should); } finally { - queryRewriteContext.setTrackRangeTimestampFromMillis(true); + queryRewriteContext.setTrackTimeRangeFilterFrom(true); } // early termination when must clause is empty and optional clauses is returning MatchNoneQueryBuilder diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 0218d1b0da06d..5ca77374dee59 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ResolvedIndices; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; @@ -23,6 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; @@ -77,8 +80,8 @@ public class QueryRewriteContext { private QueryRewriteInterceptor queryRewriteInterceptor; private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; - private Long rangeTimestampFromMillis; - private boolean trackRangeTimestampFromMillis = true; + private Long timeRangeFilterFromMillis; + private boolean trackTimeRangeFilterFrom = true; public QueryRewriteContext( final XContentParserConfiguration parserConfiguration, @@ -520,34 +523,45 @@ public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInter /** * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query */ - public Long getRangeTimestampFromMillis() { - return rangeTimestampFromMillis; + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; } /** - * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes. + * Optionally records the lower bound of a time range filter included in the query. For telemetry purposes. */ - public void setRangeTimestampFromMillis(long rangeTimestampFromMillis) { - if (trackRangeTimestampFromMillis) { - // if we got a timestamp with nanoseconds precision, round it down to millis - if (rangeTimestampFromMillis > 1_000_000_000_000_000L) { - rangeTimestampFromMillis = rangeTimestampFromMillis / 1_000_000; - } - if (this.rangeTimestampFromMillis == null) { - this.rangeTimestampFromMillis = rangeTimestampFromMillis; - } else { - // if there's more range filters on timestamp, we'll take the lowest of the lower bounds - this.rangeTimestampFromMillis = Math.min(rangeTimestampFromMillis, this.rangeTimestampFromMillis); + public void setTimeRangeFilterFromMillis(String fieldName, long timeRangeFilterFromMillis, DateFieldMapper.Resolution resolution) { + if (trackTimeRangeFilterFrom) { + if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName) || IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) { + // if we got a timestamp with nanoseconds precision, round it down to millis + if (resolution == DateFieldMapper.Resolution.NANOSECONDS) { + timeRangeFilterFromMillis = timeRangeFilterFromMillis / 1_000_000; + } + if (this.timeRangeFilterFromMillis == null) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; + } else { + // if there's more range filters on timestamp, we'll take the lowest of the lower bounds + this.timeRangeFilterFromMillis = Math.min(timeRangeFilterFromMillis, this.timeRangeFilterFromMillis); + } } } } + /** + * Records the lower bound of a time range filter included in the query. For telemetry purposes. + * Similar to {@link #setTimeRangeFilterFromMillis(String, long, DateFieldMapper.Resolution)} but used to copy the value from + * another instance of the context, that had its value previously set. + */ + public void setTimeRangeFilterFromMillis(long timeRangeFilterFromMillis) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; + } + /** * Enables or disables the tracking of the lower bound for time range filters against the @timestamp field, - * done via {@link #setRangeTimestampFromMillis(long)}. Tracking is enabled by default, and explicitly disabled to ensure - * we don't record the bound for range queries within should and must_not clauses. + * done via {@link #setTimeRangeFilterFromMillis(String, long, DateFieldMapper.Resolution)}. Tracking is enabled by default, + * and explicitly disabled to ensure we don't record the bound for range queries within should and must_not clauses. */ - public void setTrackRangeTimestampFromMillis(boolean trackRangeTimestampFromMillis) { - this.trackRangeTimestampFromMillis = trackRangeTimestampFromMillis; + public void setTrackTimeRangeFilterFrom(boolean trackTimeRangeFilterFrom) { + this.trackTimeRangeFilterFrom = trackTimeRangeFilterFrom; } } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index e37fea974fea9..f35be08dc6472 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -56,15 +56,15 @@ public void onDfsPhase(SearchContext searchContext, long tookInNanos) { @Override public void onQueryPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFromMillis = searchExecutionContext.getRangeTimestampFromMillis(); - recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFromMillis); + Long timeRangeFilterFromMillis = searchExecutionContext.getTimeRangeFilterFromMillis(); + recordPhaseLatency(queryPhaseMetric, tookInNanos, searchContext.request(), timeRangeFilterFromMillis); } @Override public void onFetchPhase(SearchContext searchContext, long tookInNanos) { SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); - Long rangeTimestampFromMillis = searchExecutionContext.getRangeTimestampFromMillis(); - recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFromMillis); + Long timeRangeFilterFromMillis = searchExecutionContext.getTimeRangeFilterFromMillis(); + recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), timeRangeFilterFromMillis); } private static void recordPhaseLatency(LongHistogram histogramMetric, long tookInNanos) { @@ -75,11 +75,11 @@ private static void recordPhaseLatency( LongHistogram histogramMetric, long tookInNanos, ShardSearchRequest request, - Long rangeTimestampFromMillis + Long timeRangeFilterFromMillis ) { Map attributes = SearchRequestAttributesExtractor.extractAttributes( request, - rangeTimestampFromMillis, + timeRangeFilterFromMillis, request.nowInMillis() ); histogramMetric.record(TimeUnit.NANOSECONDS.toMillis(tookInNanos), attributes); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java index 749bdfcd070df..ae9f14de4c24d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java @@ -73,8 +73,8 @@ public long recordTookTimeForSearchScroll(long tookTime) { return tookTime; } - public long recordTookTime(long tookTime, Long rangeTimestampFromMillis, long nowInMillis, Map attributes) { - SearchRequestAttributesExtractor.addTimeRangeAttribute(rangeTimestampFromMillis, nowInMillis, attributes); + public long recordTookTime(long tookTime, Long timeRangeFilterFromMillis, long nowInMillis, Map attributes) { + SearchRequestAttributesExtractor.addTimeRangeAttribute(timeRangeFilterFromMillis, nowInMillis, attributes); tookDurationTotalMillisHistogram.record(tookTime, attributes); return tookTime; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 2954b3cd0a4bb..2c53df81041b2 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1442,10 +1442,10 @@ private DefaultSearchContext createSearchContext( // during rewrite and normalized / evaluate templates etc. SearchExecutionContext context = new SearchExecutionContext(searchContext.getSearchExecutionContext()); Rewriteable.rewrite(request.getRewriteable(), context, true); - if (context.getRangeTimestampFromMillis() != null) { + if (context.getTimeRangeFilterFromMillis() != null) { // range queries may get rewritten to match_all or a range with open bounds. Rewriting in that case is the only place // where we parse the date and set it to the context. We need to propagate it back from the clone into the original context - searchContext.getSearchExecutionContext().setRangeTimestampFromMillis(context.getRangeTimestampFromMillis()); + searchContext.getSearchExecutionContext().setTimeRangeFilterFromMillis(context.getTimeRangeFilterFromMillis()); } assert searchContext.getSearchExecutionContext().isCacheable(); success = true; diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index fcf2c4f254cc0..96d6da79b6f7b 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -140,7 +140,7 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution // here to make sure it happens during the QUERY phase AggregationPhase.preProcess(searchContext); - addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getRangeTimestampFromMillis()); + addCollectorsAndSearch(searchContext, searchContext.getSearchExecutionContext().getTimeRangeFilterFromMillis()); RescorePhase.execute(searchContext); SuggestPhase.execute(searchContext); @@ -157,7 +157,7 @@ static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimest final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); - queryResult.setRangeTimestampFromMillis(rangeTimestampFrom); + queryResult.setTimeRangeFilterFromMillis(rangeTimestampFrom); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); diff --git a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java index c24abc53b76b3..1b3af6c22ca51 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java +++ b/server/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java @@ -81,7 +81,7 @@ public final class QuerySearchResult extends SearchPhaseResult { private final SubscribableListener aggsContextReleased; @Nullable - private Long rangeTimestampFromMillis; + private Long timeRangeFilterFromMillis; public QuerySearchResult() { this(false); @@ -460,7 +460,7 @@ private void readFromWithId(ShardSearchContextId id, StreamInput in, boolean del } } if (in.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { - rangeTimestampFromMillis = in.readOptionalLong(); + timeRangeFilterFromMillis = in.readOptionalLong(); } success = true; } finally { @@ -533,7 +533,7 @@ public void writeToNoId(StreamOutput out) throws IOException { out.writeBoolean(reduced); } if (out.getTransportVersion().supports(TIMESTAMP_RANGE_TELEMETRY)) { - out.writeOptionalLong(rangeTimestampFromMillis); + out.writeOptionalLong(timeRangeFilterFromMillis); } } @@ -587,11 +587,11 @@ private static boolean versionSupportsBatchedExecution(TransportVersion transpor return transportVersion.supports(BATCHED_QUERY_PHASE_VERSION); } - public Long getRangeTimestampFromMillis() { - return rangeTimestampFromMillis; + public Long getTimeRangeFilterFromMillis() { + return timeRangeFilterFromMillis; } - public void setRangeTimestampFromMillis(Long rangeTimestampFromMillis) { - this.rangeTimestampFromMillis = rangeTimestampFromMillis; + public void setTimeRangeFilterFromMillis(Long timeRangeFilterFromMillis) { + this.timeRangeFilterFromMillis = timeRangeFilterFromMillis; } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index aee541cbcca35..5a2232b867a03 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -137,15 +137,14 @@ private static void assertAttributes( } else { assertNull(attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); } - if (rangeOnTimestamp) { - assertEquals(rangeOnTimestamp, attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); + if (rangeOnTimestamp && rangeOnEventIngested) { + assertEquals("both", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); + } else if (rangeOnTimestamp) { + assertEquals("@timestamp", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); + } else if (rangeOnEventIngested) { + assertEquals("event.ingested", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } else { - assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_TIMESTAMP_ATTRIBUTE)); - } - if (rangeOnEventIngested) { - assertEquals(rangeOnEventIngested, attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); - } else { - assertNull(attributes.get(SearchRequestAttributesExtractor.RANGE_EVENT_INGESTED_ATTRIBUTE)); + assertEquals("none", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } } diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 8abd1de3e9337..4323541a9c8af 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -487,7 +487,7 @@ public void testTimeRangeFilterNoResults() { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); // there were no results, and no shards queried, hence no range filter extracted from the query either } @@ -539,8 +539,8 @@ private static void assertTimeRangeAttributes(Map attributes) { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } public void testTimeRangeFilterAllResultsFilterOnEventIngested() { @@ -560,11 +560,12 @@ public void testTimeRangeFilterAllResultsFilterOnEventIngested() { Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); Map attributes = measurement.attributes(); - assertEquals(4, attributes.size()); + assertEquals(5, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_event_ingested")); + assertEquals("event.ingested", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } public void testTimeRangeFilterAllResultsFilterOnEventIngestedAndTimestamp() { @@ -585,13 +586,12 @@ public void testTimeRangeFilterAllResultsFilterOnEventIngestedAndTimestamp() { Measurement measurement = measurements.getFirst(); assertEquals(searchResponse.getTook().millis(), measurement.getLong()); Map attributes = measurement.attributes(); - assertEquals(6, attributes.size()); + assertEquals(5, attributes.size()); assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_event_ingested")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp_AND_event.ingested", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { @@ -618,8 +618,8 @@ public void testTimeRangeFilterOneResultQueryAndFetchRecentTimestamps() { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("@timestamp", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("15_minutes", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("15_minutes", attributes.get("time_range_filter_from")); } public void testMultipleTimeRangeFiltersQueryAndFetchRecentTimestamps() { @@ -651,8 +651,8 @@ public void testMultipleTimeRangeFiltersQueryAndFetchRecentTimestamps() { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("@timestamp", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("1_hour", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); } public void testTimeRangeFilterAllResultsShouldClause() { @@ -714,8 +714,8 @@ public void testTimeRangeFilterAllResultsNanoPrecision() { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("1_hour", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); } public void testTimeRangeFilterAllResultsMixedPrecision() { @@ -741,8 +741,8 @@ public void testTimeRangeFilterAllResultsMixedPrecision() { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("1_hour", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("1_hour", attributes.get("time_range_filter_from")); } private void resetMeter() { diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index c30dd1e84ae65..01c47f9ad2801 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -288,9 +288,9 @@ private static void assertTimeRangeAttributes(List measurements, St assertEquals(target, attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); assertEquals(isSystem, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } } @@ -314,8 +314,9 @@ public void testTimeRangeFilterAllResults() { assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - // the range query was rewritten to one without bounds: we do track the from but we don't set the boolean flag - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + // the range query was rewritten to one without bounds: we do track the time range filter from value but we don't set + // the time range filter field because no range query is executed at the shard level. + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); // in this case, each shard queried has results to be fetched @@ -328,7 +329,7 @@ public void testTimeRangeFilterAllResults() { assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); assertEquals(false, attributes.get(SearchRequestAttributesExtractor.SYSTEM_THREAD_ATTRIBUTE_NAME)); - // no time range filter bucketing on the fetch phase, the query was rewritten to one without bounds + // no time range filter bucketing on the fetch phase, because the query was rewritten to one without bounds } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java index 4a87c42d9f2d2..f47f877209d41 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTookTimeTelemetryTests.java @@ -167,7 +167,7 @@ private static void assertAttributes(Map attributes) { assertEquals("user", attributes.get("target")); assertEquals("hits_only", attributes.get("query_type")); assertEquals("_score", attributes.get("sort")); - assertEquals(true, attributes.get("range_timestamp")); - assertEquals("older_than_14_days", attributes.get("timestamp_range_filter")); + assertEquals("@timestamp", attributes.get("time_range_filter_field")); + assertEquals("older_than_14_days", attributes.get("time_range_filter_from")); } } From 8f1e6809da7b1939dc31ca3c468d1a2c061e7817 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 20:57:39 +0200 Subject: [PATCH 27/30] iter --- .../action/search/SearchPhaseController.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index d39dd90dbc780..36f98e5a44e06 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -527,16 +527,16 @@ static ReducedQueryPhase reducedQueryPhase( sortValueFormats = result.sortValueFormats(); } - if (timeRangeFilterFromMillis == null) { - // we simply take the first one: we should get the same value from all shards anyway - timeRangeFilterFromMillis = result.getTimeRangeFilterFromMillis(); + if (result.getTimeRangeFilterFromMillis() != null) { + if (timeRangeFilterFromMillis == null) { + timeRangeFilterFromMillis = result.getTimeRangeFilterFromMillis(); + } else { + //all shards should hold the same value, besides edge cases like different mappings + // for event.ingested and @timestamp across indices being searched + timeRangeFilterFromMillis = Math.min(result.getTimeRangeFilterFromMillis(), timeRangeFilterFromMillis); + } } - assert timeRangeFilterFromMillis == null - || result.getTimeRangeFilterFromMillis() == null - || timeRangeFilterFromMillis.equals(result.getTimeRangeFilterFromMillis()) - : timeRangeFilterFromMillis + " != " + result.getTimeRangeFilterFromMillis(); - if (hasSuggest) { assert result.suggest() != null; for (Suggestion> suggestion : result.suggest()) { From 8fb2134e2850797e60dea646da19d328f11b2204 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 20:59:33 +0200 Subject: [PATCH 28/30] iter --- .../main/java/org/elasticsearch/search/query/QueryPhase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 96d6da79b6f7b..7736c05b89728 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -153,11 +153,11 @@ static void executeQuery(SearchContext searchContext) throws QueryPhaseExecution * In a package-private method so that it can be tested without having to * wire everything (mapperService, etc.) */ - static void addCollectorsAndSearch(SearchContext searchContext, Long rangeTimestampFrom) throws QueryPhaseExecutionException { + static void addCollectorsAndSearch(SearchContext searchContext, Long timeRangeFilterFromMillis) throws QueryPhaseExecutionException { final ContextIndexSearcher searcher = searchContext.searcher(); final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); - queryResult.setTimeRangeFilterFromMillis(rangeTimestampFrom); + queryResult.setTimeRangeFilterFromMillis(timeRangeFilterFromMillis); queryResult.searchTimedOut(false); try { queryResult.from(searchContext.from()); From c103fbe984b4965f8be7f2695d7bb0cfb3364b32 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 6 Oct 2025 19:10:46 +0000 Subject: [PATCH 29/30] [CI] Auto commit changes from spotless --- .../org/elasticsearch/action/search/SearchPhaseController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 36f98e5a44e06..774bb4c20faa6 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -531,7 +531,7 @@ static ReducedQueryPhase reducedQueryPhase( if (timeRangeFilterFromMillis == null) { timeRangeFilterFromMillis = result.getTimeRangeFilterFromMillis(); } else { - //all shards should hold the same value, besides edge cases like different mappings + // all shards should hold the same value, besides edge cases like different mappings // for event.ingested and @timestamp across indices being searched timeRangeFilterFromMillis = Math.min(result.getTimeRangeFilterFromMillis(), timeRangeFilterFromMillis); } From 260da9ef4eeef68b68ea4742cdc0dd2111db2525 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 6 Oct 2025 21:46:52 +0200 Subject: [PATCH 30/30] iter --- .../search/SearchRequestAttributesExtractorTests.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java index 5a2232b867a03..cbb684acb39bf 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestAttributesExtractorTests.java @@ -138,13 +138,16 @@ private static void assertAttributes( assertNull(attributes.get(SearchRequestAttributesExtractor.KNN_ATTRIBUTE)); } if (rangeOnTimestamp && rangeOnEventIngested) { - assertEquals("both", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); + assertEquals( + "@timestamp_AND_event.ingested", + attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE) + ); } else if (rangeOnTimestamp) { assertEquals("@timestamp", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } else if (rangeOnEventIngested) { assertEquals("event.ingested", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } else { - assertEquals("none", attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); + assertNull(attributes.get(SearchRequestAttributesExtractor.TIME_RANGE_FILTER_FIELD_ATTRIBUTE)); } }