From 65c255395eb443c56b8d0b9736d65d41734c6907 Mon Sep 17 00:00:00 2001 From: ForVic Date: Wed, 6 Aug 2025 14:30:49 -0700 Subject: [PATCH 1/3] [LIHADOOP-81008] Add a Driver Metrics Polling Interval (#462) --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- .../scala/org/apache/spark/internal/config/package.scala | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index ce62f8f5d5473..a7dbf01fc9978 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -614,7 +614,7 @@ class SparkContext(config: SparkConf) extends Logging { _heartbeater = new Heartbeater( () => SparkContext.this.reportHeartBeat(_executorMetricsSource), "driver-heartbeater", - conf.get(EXECUTOR_HEARTBEAT_INTERVAL)) + conf.get(DRIVER_HEARTBEAT_INTERVAL)) _heartbeater.start() // start TaskScheduler after taskScheduler sets DAGScheduler reference in DAGScheduler's diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index c25a4fd45c58a..81805ddbd03e5 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -359,6 +359,11 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createWithDefaultString("10s") + private[spark] val DRIVER_HEARTBEAT_INTERVAL = + ConfigBuilder("spark.driver.heartbeatInterval") + .version("3.1.1") + .fallbackConf(EXECUTOR_HEARTBEAT_INTERVAL) + private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES = ConfigBuilder("spark.executor.heartbeat.maxFailures") .internal() From 4402779600b53cde9534411af8e60e363493b6f5 Mon Sep 17 00:00:00 2001 From: ForVic Date: Wed, 6 Aug 2025 14:33:57 -0700 Subject: [PATCH 2/3] change version introduced --- .../main/scala/org/apache/spark/internal/config/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 81805ddbd03e5..f335307c47bfc 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -361,7 +361,7 @@ package object config { private[spark] val DRIVER_HEARTBEAT_INTERVAL = ConfigBuilder("spark.driver.heartbeatInterval") - .version("3.1.1") + .version("4.1.0") .fallbackConf(EXECUTOR_HEARTBEAT_INTERVAL) private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES = From ec588a051fdb8273743d8c5d0433294bbe157e47 Mon Sep 17 00:00:00 2001 From: ForVic Date: Tue, 26 Aug 2025 14:07:25 -0700 Subject: [PATCH 3/3] Rename config to spark.driver.metrics.pollingInterval --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index a7dbf01fc9978..6352a1f3f5d1d 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -614,7 +614,7 @@ class SparkContext(config: SparkConf) extends Logging { _heartbeater = new Heartbeater( () => SparkContext.this.reportHeartBeat(_executorMetricsSource), "driver-heartbeater", - conf.get(DRIVER_HEARTBEAT_INTERVAL)) + conf.get(DRIVER_METRICS_POLLING_INTERVAL)) _heartbeater.start() // start TaskScheduler after taskScheduler sets DAGScheduler reference in DAGScheduler's diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index f335307c47bfc..3b8974f806e1e 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -359,8 +359,8 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createWithDefaultString("10s") - private[spark] val DRIVER_HEARTBEAT_INTERVAL = - ConfigBuilder("spark.driver.heartbeatInterval") + private[spark] val DRIVER_METRICS_POLLING_INTERVAL = + ConfigBuilder("spark.driver.metrics.pollingInterval") .version("4.1.0") .fallbackConf(EXECUTOR_HEARTBEAT_INTERVAL)