From fb873aa3de1ca4b814330f786ce4bc459d22597f Mon Sep 17 00:00:00 2001 From: Denis Policastro Date: Thu, 25 Jul 2024 14:13:32 -0300 Subject: [PATCH] feat: update grafana dashboard --- grafana/dashboards/dash_charon_overview.json | 437 +++++++++---------- 1 file changed, 212 insertions(+), 225 deletions(-) diff --git a/grafana/dashboards/dash_charon_overview.json b/grafana/dashboards/dash_charon_overview.json index 794d1f40..7f37dc2f 100644 --- a/grafana/dashboards/dash_charon_overview.json +++ b/grafana/dashboards/dash_charon_overview.json @@ -24,15 +24,14 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, + "id": 2, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": true, "keepTime": true, - "tags": [ - "cluster-labels" - ], + "tags": ["cluster-labels"], "targetBlank": false, "title": "", "tooltip": "", @@ -70,7 +69,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", "fieldConfig": { @@ -171,9 +170,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -181,12 +178,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -205,7 +202,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Successful cluster duty percentage by type over the whole time range", "fieldConfig": { @@ -250,9 +247,7 @@ "minVizWidth": 200, "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": true }, @@ -261,12 +256,12 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -284,7 +279,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", "fieldConfig": { @@ -328,9 +323,7 @@ "minVizWidth": 200, "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -338,12 +331,12 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -356,7 +349,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range])) by (le)),1))\n )\n)", @@ -392,7 +385,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", "fieldConfig": { @@ -466,9 +459,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -480,12 +471,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -500,7 +491,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -513,7 +504,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -527,7 +518,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -541,7 +532,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -559,7 +550,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "This panels shows any health checks that are failing. See for help troubleshooting. \n\nNote the different severities:\n- **info**: Only informational, might not be an actual problem.\n- **warning**: Performance is degraded, operator intervention only required if this persists for a long time.\n- **critical**: Significant problem, validators probably not operation at all. Immediate operator intervention required.\n \n\n", "fieldConfig": { @@ -669,19 +660,17 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -716,7 +705,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (👍) or *relay* (👎) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", "fieldConfig": { @@ -1139,9 +1128,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "frameIndex": 0, @@ -1153,12 +1140,12 @@ } ] }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1171,7 +1158,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1185,7 +1172,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1200,7 +1187,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1215,7 +1202,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1230,7 +1217,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1244,7 +1231,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1258,7 +1245,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1273,7 +1260,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1288,7 +1275,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1303,7 +1290,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1318,7 +1305,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "disableTextWrap": false, "editorMode": "code", @@ -1438,7 +1425,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "", "fieldConfig": { @@ -1499,7 +1486,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1514,7 +1501,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1529,7 +1516,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1548,7 +1535,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", "fieldConfig": { @@ -1701,7 +1688,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1720,7 +1707,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Type of connections to each peer in the cluster during the time window:\n - 🟢 Green: Direct connection (this is the preferred)\n - 🟡 Yellow: Indirect relay connection (this is ok if only temporary)\n - 🔴 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", "fieldConfig": { @@ -1787,7 +1774,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", @@ -1804,7 +1791,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "", "fieldConfig": { @@ -1821,7 +1808,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -1899,9 +1887,7 @@ "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -1913,12 +1899,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1932,7 +1918,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1948,7 +1934,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1962,7 +1948,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1976,7 +1962,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -1990,7 +1976,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2004,7 +1990,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2018,7 +2004,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2036,7 +2022,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"❗️Peer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", "fieldConfig": { @@ -2056,7 +2042,8 @@ "mode": "absolute", "steps": [ { - "color": "light-blue" + "color": "light-blue", + "value": null }, { "color": "red", @@ -2220,12 +2207,8 @@ "cellHeight": "sm", "footer": { "countRows": false, - "fields": [ - "Value #B" - ], - "reducer": [ - "sum" - ], + "fields": ["Value #B"], + "reducer": ["sum"], "show": true }, "showHeader": true, @@ -2236,12 +2219,12 @@ } ] }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2257,7 +2240,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2272,7 +2255,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2337,7 +2320,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", "fieldConfig": { @@ -2391,7 +2374,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2425,7 +2409,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", @@ -2438,7 +2422,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", @@ -2455,7 +2439,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (👍) or *relay* (👎) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Uptime**\" The duration since peer was restarted.\n- **ClockDiff**: Difference between local and peer's clock time. More than 2s is bad.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", "fieldConfig": { @@ -2479,7 +2463,8 @@ "mode": "absolute", "steps": [ { - "color": "transparent" + "color": "transparent", + "value": null } ] } @@ -2770,7 +2755,8 @@ "mode": "absolute", "steps": [ { - "color": "#585858" + "color": "#585858", + "value": null }, { "color": "super-light-green", @@ -2807,9 +2793,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "frameIndex": 0, @@ -2821,12 +2805,12 @@ } ] }, - "pluginVersion": "10.4.2", + "pluginVersion": "10.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2839,7 +2823,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2853,7 +2837,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2868,7 +2852,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2882,7 +2866,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2897,7 +2881,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2912,7 +2896,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2927,7 +2911,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2941,7 +2925,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2955,7 +2939,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2970,7 +2954,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -2985,7 +2969,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -3113,7 +3097,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Completed duties by type over time.", "fieldConfig": { @@ -3173,7 +3157,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -3191,7 +3175,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Successful duty participation per peer (filtered by selected duty) over time.", "fieldConfig": { @@ -3250,7 +3234,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -3267,7 +3251,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Failed duties by type over time. Use data link to show logs.", "fieldConfig": { @@ -3329,7 +3313,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -3347,7 +3331,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Missed duty participation per peer (filtered by selected duty) over time.", "fieldConfig": { @@ -3406,7 +3390,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -3423,7 +3407,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Failed duties percentage by type over time", "fieldConfig": { @@ -3502,7 +3486,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -3520,7 +3504,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Missed duty participation per duty (filtered by peer) over time.", "fieldConfig": { @@ -3581,7 +3565,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -3598,7 +3582,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", "fieldConfig": { @@ -3676,7 +3660,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", @@ -3693,7 +3677,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Which operators are having a growing number of relay registrations failures, indicating there may be a risk to an upcoming MEV block production opportunity. Be conscious that absence of an operator from this list may mean their MEV registration is working, or it may mean it is not enabled at all and thus not failing. ", "fieldConfig": { @@ -3797,9 +3781,7 @@ "id": 242, "options": { "legend": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true, @@ -3816,7 +3798,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(increase(app_eth2_errors_total{endpoint=~\"submit_validator_registrations\", cluster_name=~\"$cluster_name\"}[$__range])) by (cluster_peer, cluster_name)", @@ -3832,7 +3814,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", "fieldConfig": { @@ -3911,7 +3893,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -3929,7 +3911,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4018,7 +4000,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -4031,7 +4013,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -4049,7 +4031,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "The delay after the start of the slot when the duty was successfully broadcasted. ", "fieldConfig": { @@ -4137,7 +4119,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(histogram_quantile(0.90, rate(core_bcast_broadcast_delay_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) ", @@ -4154,7 +4136,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4241,7 +4223,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(histogram_quantile(0.90, increase(core_consensus_duration_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]))) by (timer)", @@ -4258,7 +4240,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4344,7 +4326,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer)", @@ -4361,7 +4343,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of consensus timeouts per time window.", "fieldConfig": { @@ -4437,7 +4419,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", @@ -4464,7 +4446,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4542,7 +4524,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -4555,7 +4537,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", @@ -4571,7 +4553,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4650,7 +4632,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", @@ -4669,7 +4651,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -4760,7 +4742,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -4773,7 +4755,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", @@ -4790,7 +4772,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Total time spent on garbage collection per interval", "fieldConfig": { @@ -4870,7 +4852,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", @@ -4906,7 +4888,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "", "fieldConfig": { @@ -4945,9 +4927,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "frameIndex": 0, @@ -4958,7 +4938,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": false, @@ -4975,9 +4955,7 @@ { "id": "labelsToFields", "options": { - "keepLabels": [ - "version" - ], + "keepLabels": ["version"], "mode": "rows" } } @@ -4987,7 +4965,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Upstream beacon node API request rate per second by endpoint", "fieldConfig": { @@ -5065,7 +5043,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5082,7 +5060,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Upstream beacon node API request latency (90th percentile) by endpoint by timewindow", "fieldConfig": { @@ -5160,7 +5138,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5177,7 +5155,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of errors per endpoint per time window", "fieldConfig": { @@ -5233,9 +5211,7 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [ - "submit_beacon_block" - ], + "names": ["submit_beacon_block"], "prefix": "All except:", "readOnly": true } @@ -5276,7 +5252,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5293,7 +5269,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Beacon node score over time", "fieldConfig": { @@ -5379,7 +5355,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", @@ -5395,7 +5371,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of consensus layer P2P peers the beacon node is connected to", "fieldConfig": { @@ -5478,7 +5454,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", @@ -5504,7 +5480,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -5581,7 +5557,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5598,7 +5574,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -5675,7 +5651,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5692,7 +5668,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of errors per endpoint per time window", "fieldConfig": { @@ -5766,7 +5742,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5800,7 +5776,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Type of connections to each peer in the cluster during the time window:\n - 🟢 Green: Direct connection (this is the preferred)\n - 🟡 Yellow: Indirect relay connection (this is ok if only temporary)\n - 🔴 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", "fieldConfig": { @@ -5860,7 +5836,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", @@ -5877,7 +5853,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -5963,7 +5939,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -5980,7 +5956,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Time windows that contain ping errors per peer", "fieldConfig": { @@ -6031,7 +6007,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", @@ -6047,7 +6023,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -6114,7 +6090,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -6131,7 +6107,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -6209,7 +6185,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -6226,7 +6202,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Number of new libp2p connections per peer per time window", "fieldConfig": { @@ -6305,7 +6281,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "exemplar": true, @@ -6322,7 +6298,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "", "fieldConfig": { @@ -6412,7 +6388,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", @@ -6428,7 +6404,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", "fieldConfig": { @@ -6519,7 +6495,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", @@ -6535,7 +6511,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "fieldConfig": { "defaults": { @@ -6585,7 +6561,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", @@ -6601,7 +6577,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", "fieldConfig": { @@ -6692,7 +6668,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer)", @@ -6708,7 +6684,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", "fieldConfig": { @@ -6787,7 +6763,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "editorMode": "code", "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", @@ -6804,21 +6780,32 @@ "refresh": "1m", "revision": 1, "schemaVersion": 39, - "tags": [ - "cluster-labels", - "charon" - ], + "tags": ["cluster-labels", "charon"], "templating": { "list": [ { "current": { "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] }, "description": "", "hide": 0, @@ -6866,12 +6853,12 @@ { "current": { "selected": false, - "text": "test-lido-cluster", - "value": "test-lido-cluster" + "text": "Lido x Obol: Affable Alligator", + "value": "Lido x Obol: Affable Alligator" }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "definition": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", "description": "", @@ -6894,12 +6881,12 @@ { "current": { "selected": false, - "text": "41eb9d9", - "value": "41eb9d9" + "text": "5da2126", + "value": "5da2126" }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", "description": "", @@ -6922,12 +6909,12 @@ { "current": { "selected": false, - "text": "thoughtful-sister", - "value": "thoughtful-sister" + "text": "modern-painter", + "value": "modern-painter" }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", "description": "", @@ -6955,7 +6942,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", "description": "Prometheus job (can be ignored)", @@ -6984,7 +6971,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$datasource" }, "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", "description": "", @@ -7173,6 +7160,6 @@ "timezone": "", "title": "Charon Overview", "uid": "d6qujIJVk", - "version": 3, + "version": 1, "weekStart": "" }