Add time range bucketing attribute to APM took time latency metrics

docs/changelog/135549.yaml

@@ -0,0 +1,5 @@
+pr: 135549
+summary: Add time range bucketing attribute to APM took time latency metrics
+area: Search
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/search/CountOnlyQueryPhaseResultConsumer.java

@@ -86,7 +86,8 @@ public SearchPhaseController.ReducedQueryPhase reduce() throws Exception {
             1,
             0,
             0,
-            results.isEmpty()
+            results.isEmpty(),
+            null
         );
         if (progressListener != SearchProgressListener.NOOP) {
             progressListener.notifyFinalReduce(

server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java

@@ -251,7 +251,8 @@ private SearchPhaseController.ReducedQueryPhase newReducedQueryPhaseResults(
             reducedQueryPhase.numReducePhases(),
             reducedQueryPhase.size(),
             reducedQueryPhase.from(),
-            reducedQueryPhase.isEmptyResult()
+            reducedQueryPhase.isEmptyResult(),
+            reducedQueryPhase.rangeTimestampFrom()
         );
     }
 

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -487,7 +487,8 @@ static ReducedQueryPhase reducedQueryPhase(
                 numReducePhases,
                 0,
                 0,
-                true
+                true,
+                null
             );
         }
         final List<QuerySearchResult> nonNullResults = new ArrayList<>();
@@ -516,6 +517,7 @@ static ReducedQueryPhase reducedQueryPhase(
             : Collections.emptyMap();
         int from = 0;
         int size = 0;
+        Long rangeTimestampFrom = null;
         DocValueFormat[] sortValueFormats = null;
         for (QuerySearchResult result : nonNullResults) {
             from = result.from();
@@ -525,6 +527,11 @@ static ReducedQueryPhase reducedQueryPhase(
                 sortValueFormats = result.sortValueFormats();
             }
 
+            if (rangeTimestampFrom == null) {
+                // we simply take the first one: we should get the same value from all shards anyways
+                rangeTimestampFrom = result.getRangeTimestampFrom();
+            }
+
             if (hasSuggest) {
                 assert result.suggest() != null;
                 for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : result.suggest()) {
@@ -579,7 +586,8 @@ static ReducedQueryPhase reducedQueryPhase(
             numReducePhases,
             size,
             from,
-            false
+            false,
+            rangeTimestampFrom
         );
     }
 
@@ -662,7 +670,8 @@ public record ReducedQueryPhase(
         // the offset into the merged top hits
         int from,
         // <code>true</code> iff the query phase had no results. Otherwise <code>false</code>
-        boolean isEmptyResult
+        boolean isEmptyResult,
+        Long rangeTimestampFrom
     ) {
 
         public ReducedQueryPhase {
@@ -683,7 +692,8 @@ public SearchResponseSections buildResponse(SearchHits hits, Collection<? extend
                 timedOut,
                 terminatedEarly,
                 buildSearchProfileResults(fetchResults),
-                numReducePhases
+                numReducePhases,
+                rangeTimestampFrom
             );
         }
 

server/src/main/java/org/elasticsearch/action/search/SearchRequestAttributesExtractor.java

@@ -343,6 +343,13 @@ private enum TimeRangeBucket {
         }
     }
 
+    public static void addTimeRangeAttribute(Long timeRangeFrom, long nowInMillis, Map<String, Object> attributes) {
+        if (timeRangeFrom != null) {
+            String timestampRangeFilter = introspectTimeRange(timeRangeFrom, nowInMillis);
+            attributes.put(TIMESTAMP_RANGE_FILTER_ATTRIBUTE, timestampRangeFilter);
+        }
+    }
+
     static String introspectTimeRange(long timeRangeFrom, long nowInMillis) {
         for (TimeRangeBucket value : TimeRangeBucket.values()) {
             if (timeRangeFrom >= nowInMillis - value.millis) {

server/src/main/java/org/elasticsearch/action/search/SearchResponse.java

@@ -87,6 +87,8 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO
     private final ShardSearchFailure[] shardFailures;
     private final Clusters clusters;
     private final long tookInMillis;
+    // only used for telemetry purposes on the coordinating node, where the search response gets created
+    private transient Long rangeTimestampFrom;
 
     private final RefCounted refCounted = LeakTracker.wrap(new SimpleRefCounted());
 
@@ -187,6 +189,7 @@ public SearchResponse(
             clusters,
             pointInTimeId
         );
+        this.rangeTimestampFrom = searchResponseSections.rangeTimestampFrom;
     }
 
     public SearchResponse(
@@ -464,6 +467,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalBytesReference(pointInTimeId);
     }
 
+    public Long getRangeTimestampFrom() {
+        return rangeTimestampFrom;
+    }
+
     @Override
     public String toString() {
         return hasReferences() == false ? "SearchResponse[released]" : Strings.toString(this);

server/src/main/java/org/elasticsearch/action/search/SearchResponseSections.java

@@ -32,7 +32,8 @@ public class SearchResponseSections implements Releasable {
         false,
         null,
         null,
-        1
+        1,
+        null
     );
     public static final SearchResponseSections EMPTY_WITHOUT_TOTAL_HITS = new SearchResponseSections(
         SearchHits.EMPTY_WITHOUT_TOTAL_HITS,
@@ -41,7 +42,8 @@ public class SearchResponseSections implements Releasable {
         false,
         null,
         null,
-        1
+        1,
+        null
     );
     protected final SearchHits hits;
     protected final InternalAggregations aggregations;
@@ -50,6 +52,7 @@ public class SearchResponseSections implements Releasable {
     protected final boolean timedOut;
     protected final Boolean terminatedEarly;
     protected final int numReducePhases;
+    protected final Long rangeTimestampFrom;
 
     public SearchResponseSections(
         SearchHits hits,
@@ -58,7 +61,8 @@ public SearchResponseSections(
         boolean timedOut,
         Boolean terminatedEarly,
         SearchProfileResults profileResults,
-        int numReducePhases
+        int numReducePhases,
+        Long rangeTimestampFrom
     ) {
         this.hits = hits;
         this.aggregations = aggregations;
@@ -67,6 +71,7 @@ public SearchResponseSections(
         this.timedOut = timedOut;
         this.terminatedEarly = terminatedEarly;
         this.numReducePhases = numReducePhases;
+        this.rangeTimestampFrom = rangeTimestampFrom;
     }
 
     public final SearchHits hits() {

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -430,7 +430,12 @@ public void onFailure(Exception e) {
                     Arrays.stream(resolvedIndices.getConcreteLocalIndices()).map(Index::getName).toArray(String[]::new)
                 );
                 if (collectCCSTelemetry == false || resolvedIndices.getRemoteClusterIndices().isEmpty()) {
-                    searchResponseActionListener = new SearchTelemetryListener(delegate, searchResponseMetrics, searchRequestAttributes);
+                    searchResponseActionListener = new SearchTelemetryListener(
+                        delegate,
+                        searchResponseMetrics,
+                        searchRequestAttributes,
+                        timeProvider.absoluteStartMillis()
+                    );
                 } else {
                     CCSUsage.Builder usageBuilder = new CCSUsage.Builder();
                     usageBuilder.setRemotesCount(resolvedIndices.getRemoteClusterIndices().size());
@@ -459,6 +464,7 @@ public void onFailure(Exception e) {
                         delegate,
                         searchResponseMetrics,
                         searchRequestAttributes,
+                        timeProvider.absoluteStartMillis(),
                         usageService,
                         usageBuilder
                     );
@@ -2046,6 +2052,7 @@ static String[] ignoreBlockedIndices(ProjectState projectState, String[] concret
     private static class SearchTelemetryListener extends DelegatingActionListener<SearchResponse, SearchResponse> {
         private final CCSUsage.Builder usageBuilder;
         private final SearchResponseMetrics searchResponseMetrics;
+        private final long nowInMillis;
         private final UsageService usageService;
         private final boolean collectCCSTelemetry;
         private final Map<String, Object> searchRequestAttributes;
@@ -2054,12 +2061,14 @@ private static class SearchTelemetryListener extends DelegatingActionListener<Se
             ActionListener<SearchResponse> listener,
             SearchResponseMetrics searchResponseMetrics,
             Map<String, Object> searchRequestAttributes,
+            long nowInMillis,
             UsageService usageService,
             CCSUsage.Builder usageBuilder
         ) {
             super(listener);
             this.searchResponseMetrics = searchResponseMetrics;
             this.searchRequestAttributes = searchRequestAttributes;
+            this.nowInMillis = nowInMillis;
             this.collectCCSTelemetry = true;
             this.usageService = usageService;
             this.usageBuilder = usageBuilder;
@@ -2068,11 +2077,13 @@ private static class SearchTelemetryListener extends DelegatingActionListener<Se
         SearchTelemetryListener(
             ActionListener<SearchResponse> listener,
             SearchResponseMetrics searchResponseMetrics,
-            Map<String, Object> searchRequestAttributes
+            Map<String, Object> searchRequestAttributes,
+            long nowInMillis
         ) {
             super(listener);
             this.searchResponseMetrics = searchResponseMetrics;
             this.searchRequestAttributes = searchRequestAttributes;
+            this.nowInMillis = nowInMillis;
             this.collectCCSTelemetry = false;
             this.usageService = null;
             this.usageBuilder = null;
@@ -2081,7 +2092,12 @@ private static class SearchTelemetryListener extends DelegatingActionListener<Se
         @Override
         public void onResponse(SearchResponse searchResponse) {
             try {
-                searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis(), searchRequestAttributes);
+                searchResponseMetrics.recordTookTime(
+                    searchResponse.getTookInMillis(),
+                    searchResponse.getRangeTimestampFrom(),
+                    nowInMillis,
+                    searchRequestAttributes
+                );
                 SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus =
                     SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS;
                 if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) {

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -944,7 +944,7 @@ public Relation isFieldWithinQuery(
                     minValue = Long.min(minValue, skipper.minValue());
                     maxValue = Long.max(maxValue, skipper.maxValue());
                 }
-                return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context);
+                return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name());
             }
             byte[] minPackedValue = PointValues.getMinPackedValue(reader, name());
             if (minPackedValue == null) {
@@ -954,7 +954,7 @@ public Relation isFieldWithinQuery(
             long minValue = LongPoint.decodeDimension(minPackedValue, 0);
             long maxValue = LongPoint.decodeDimension(PointValues.getMaxPackedValue(reader, name()), 0);
 
-            return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context);
+            return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context, name());
         }
 
         public DateMathParser resolveDateMathParser(DateMathParser dateParser, Object from, Object to) {
@@ -977,7 +977,8 @@ public Relation isFieldWithinQuery(
             boolean includeUpper,
             ZoneId timeZone,
             DateMathParser dateParser,
-            QueryRewriteContext context
+            QueryRewriteContext context,
+            String fieldName
         ) {
             dateParser = resolveDateMathParser(dateParser, from, to);
 
@@ -990,6 +991,9 @@ public Relation isFieldWithinQuery(
                     }
                     ++fromInclusive;
                 }
+                if (fieldName.equals(DataStream.TIMESTAMP_FIELD_NAME)) {
+                    context.setRangeTimestampFrom(fromInclusive);
+                }
             }
 
             long toInclusive = Long.MAX_VALUE;

server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java

@@ -300,8 +300,13 @@ public String getWriteableName() {
     protected Query doToQuery(SearchExecutionContext context) throws IOException {
         BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
         addBooleanClauses(context, booleanQueryBuilder, mustClauses, BooleanClause.Occur.MUST);
-        addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT);
-        addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD);
+        try {
+            context.setTrackRangeTimestampFrom(false);
+            addBooleanClauses(context, booleanQueryBuilder, mustNotClauses, BooleanClause.Occur.MUST_NOT);
+            addBooleanClauses(context, booleanQueryBuilder, shouldClauses, BooleanClause.Occur.SHOULD);
+        } finally {
+            context.setTrackRangeTimestampFrom(true);
+        }
         addBooleanClauses(context, booleanQueryBuilder, filterClauses, BooleanClause.Occur.FILTER);
         BooleanQuery booleanQuery = booleanQueryBuilder.build();
         if (booleanQuery.clauses().isEmpty()) {
@@ -348,9 +353,21 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
             return new MatchAllQueryBuilder().boost(boost()).queryName(queryName());
         }
         changed |= rewriteClauses(queryRewriteContext, mustClauses, newBuilder::must);
-        changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot);
+
+        try {
+            queryRewriteContext.setTrackRangeTimestampFrom(false);
+            changed |= rewriteClauses(queryRewriteContext, mustNotClauses, newBuilder::mustNot);
+        } finally {
+            queryRewriteContext.setTrackRangeTimestampFrom(true);
+        }
         changed |= rewriteClauses(queryRewriteContext, filterClauses, newBuilder::filter);
-        changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should);
+        try {
+            queryRewriteContext.setTrackRangeTimestampFrom(false);
+            changed |= rewriteClauses(queryRewriteContext, shouldClauses, newBuilder::should);
+        } finally {
+            queryRewriteContext.setTrackRangeTimestampFrom(true);
+        }
+
         // early termination when must clause is empty and optional clauses is returning MatchNoneQueryBuilder
         if (mustClauses.size() == 0
             && filterClauses.size() == 0

server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -78,6 +78,8 @@ public class QueryRewriteContext {
     private QueryRewriteInterceptor queryRewriteInterceptor;
     private final Boolean ccsMinimizeRoundTrips;
     private final boolean isExplain;
+    private Long rangeTimestampFrom;
+    private boolean trackRangeTimestampFrom = true;
 
     public QueryRewriteContext(
         final XContentParserConfiguration parserConfiguration,
@@ -520,4 +522,32 @@ public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInter
         this.queryRewriteInterceptor = queryRewriteInterceptor;
     }
 
+    /**
+     * Returns the minimum lower bound across the time ranges filters against the @timestamp field included in the query
+     */
+    public Long getRangeTimestampFrom() {
+        return rangeTimestampFrom;
+    }
+
+    /**
+     * Records the lower bound of a time range filter against the @timestamp field included in the query. For telemetry purposes.
+     */
+    public void setRangeTimestampFrom(long rangeTimestampFrom) {
+        if (trackRangeTimestampFrom) {
+            if (this.rangeTimestampFrom == null) {
+                this.rangeTimestampFrom = rangeTimestampFrom;
+            } else {
+                this.rangeTimestampFrom = Math.min(rangeTimestampFrom, this.rangeTimestampFrom);
+            }
+        }
+    }
+
+    /**
+     * Enables or disables the tracking of the lower bound for time range filters against the @timestamp field,
+     * done via {@link #setRangeTimestampFrom(long)}. Tracking is enabled by default, and explicitly disabled to ensure
+     * we don't record the bound for range queries within should and must_not clauses.
+     */
+    public void setTrackRangeTimestampFrom(boolean trackRangeTimestampFrom) {
+        this.trackRangeTimestampFrom = trackRangeTimestampFrom;
+    }
 }

server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java

@@ -448,7 +448,8 @@ protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext c
                 includeUpper,
                 timeZone,
                 dateMathParser,
-                coordinatorRewriteContext
+                coordinatorRewriteContext,
+                dateFieldType.name()
             );
         }
         // If the field type is null or not of type DataFieldType then we have no idea whether this range query will match during