feat: add custommetrics cron for SLO monitoring (#4288)

michaelkedar · gemini-code-assist[bot] · web-flow · commit 1874e214add6 · 2025-11-05T14:53:13.000+11:00
Since it's seemingly impossible to compute this in GCP, add a
`custommetrics` cronjob to compute time since last export as a metric to
use as an SLO policy.
Also, added missing resource/request limits to the recoverer and updated
exporter's cronLastSuccessfulTimeMins to 30.

---------

Co-authored-by: gemini-code-assist[bot] &lt;176961590+gemini-code-assist[bot]@users.noreply.github.com&gt;
diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml
@@ -98,7 +98,7 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/recoverer']
   waitFor: ['build-recoverer', 'cloud-build-queue']
 
-# Build/push exporter/record-checker go images
+# Build/push go images
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: 'bash'
   args: ['-c', 'docker pull gcr.io/oss-vdb/exporter:latest || exit 0']
@@ -127,6 +127,20 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/record-checker']
   waitFor: ['build-record-checker', 'cloud-build-queue']
 
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'bash'
+  args: ['-c', 'docker pull gcr.io/oss-vdb/custommetrics:latest || exit 0']
+  id: 'pull-custommetrics'
+  waitFor: ['setup']
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/oss-vdb/custommetrics:latest', '-t', 'gcr.io/oss-vdb/custommetrics:$COMMIT_SHA', '-f', 'cmd/custommetrics/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/custommetrics:latest', '--pull', '.']
+  dir: 'go'
+  id: 'build-custommetrics'
+  waitFor: ['pull-custommetrics']
+- name: gcr.io/cloud-builders/docker
+  args: ['push', '--all-tags', 'gcr.io/oss-vdb/custommetrics']
+  waitFor: ['build-custommetrics', 'cloud-build-queue']
+
 # Build/push staging-api-test images to gcr.io/oss-vdb-test.
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.']
@@ -337,7 +351,8 @@ steps:
      nvd-mirror=gcr.io/oss-vdb/nvd-mirror:$COMMIT_SHA,\
      recoverer=gcr.io/oss-vdb/recoverer:$COMMIT_SHA,\
      cve5-to-osv=gcr.io/oss-vdb/cve5-to-osv:$COMMIT_SHA,\
-     record-checker=gcr.io/oss-vdb/record-checker:$COMMIT_SHA"
+     record-checker=gcr.io/oss-vdb/record-checker:$COMMIT_SHA,\
+     custommetrics=gcr.io/oss-vdb/custommetrics:$COMMIT_SHA"
   ]
   dir: deployment/clouddeploy/gke-workers
 
@@ -396,3 +411,4 @@ images:
 - 'gcr.io/oss-vdb/recoverer:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/cve5-to-osv:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/record-checker:$COMMIT_SHA'
+- 'gcr.io/oss-vdb/custommetrics:$COMMIT_SHA'
diff --git a/deployment/clouddeploy/gke-workers/base/custommetrics.yaml b/deployment/clouddeploy/gke-workers/base/custommetrics.yaml
@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: custommetrics
+  labels:
+    cronLastSuccessfulTimeMins: "5"
+spec:
+  schedule: "* * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: custommetrics
+            image: custommetrics
+            imagePullPolicy: Always
+            resources:
+              requests:
+                cpu: "1"
+                memory: "256Mi"
+              limits:
+                cpu: "2"
+                memory: "512Mi"
+          restartPolicy: Never
diff --git a/deployment/clouddeploy/gke-workers/base/exporter.yaml b/deployment/clouddeploy/gke-workers/base/exporter.yaml
@@ -3,7 +3,7 @@ kind: CronJob
 metadata:
   name: exporter
   labels:
-    cronLastSuccessfulTimeMins: "180"
+    cronLastSuccessfulTimeMins: "30"
 spec:
   schedule: "*/15 * * * *"
   concurrencyPolicy: Forbid
diff --git a/deployment/clouddeploy/gke-workers/base/kustomization.yaml b/deployment/clouddeploy/gke-workers/base/kustomization.yaml
@@ -27,3 +27,5 @@ resources:
 - recoverer.yaml
 - record-checker.yaml
 - cve5-to-osv.yaml
+- custommetrics.yaml
+
diff --git a/deployment/clouddeploy/gke-workers/base/recoverer.yaml b/deployment/clouddeploy/gke-workers/base/recoverer.yaml
@@ -30,3 +30,11 @@ spec:
       - name: recoverer
         image: recoverer
         imagePullPolicy: Always
+      resources:
+        requests:
+          cpu: "10m"
+          memory: "256Mi"
+        limits:
+          cpu: "200m"
+          memory: "512Mi"
+
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/custommetrics.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/custommetrics.yaml
@@ -0,0 +1,15 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: custommetrics
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: custommetrics
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb-test
+
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml
@@ -23,3 +23,4 @@ patches:
 - path: generate-sitemap.yaml
 - path: recoverer.yaml
 - path: record-checker.yaml
+- path: custommetrics.yaml
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/custommetrics.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/custommetrics.yaml
@@ -0,0 +1,15 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: custommetrics
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: custommetrics
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb
+
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml
@@ -22,3 +22,5 @@ patches:
 - path: recoverer.yaml
 - path: record-checker.yaml
 - path: cve5-to-osv.yaml
+- path: custommetrics.yaml
+
diff --git a/go/cmd/custommetrics/Dockerfile b/go/cmd/custommetrics/Dockerfile
@@ -0,0 +1,31 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.25.3-alpine@sha256:aee43c3ccbf24fdffb7295693b6e33b21e01baec1b2a55acc351fde345e9ec34 AS build
+
+WORKDIR /src
+
+COPY ./go.mod /src/go.mod
+COPY ./go.sum /src/go.sum
+RUN go mod download && go mod verify
+
+
+COPY ./ /src/
+RUN CGO_ENABLED=0 go build -o custommetrics ./cmd/custommetrics
+
+FROM gcr.io/distroless/static-debian12@sha256:87bce11be0af225e4ca761c40babb06d6d559f5767fbf7dc3c47f0f1a466b92c
+
+COPY --from=build /src/custommetrics /
+
+ENTRYPOINT ["/custommetrics"]
diff --git a/go/cmd/custommetrics/main.go b/go/cmd/custommetrics/main.go
@@ -0,0 +1,112 @@
+// Package main for custommetrics calculates and exports custom metrics to Google Cloud Monitoring.
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"time"
+
+	monitoring "cloud.google.com/go/monitoring/apiv3/v2"
+	"cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
+	"github.com/google/osv.dev/go/logger"
+	"google.golang.org/api/iterator"
+	"google.golang.org/genproto/googleapis/api/metric"
+	"google.golang.org/genproto/googleapis/api/monitoredres"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	kubeStateMetricLastRun    = "prometheus.googleapis.com/kube_cronjob_status_last_successful_time/gauge"
+	customMetricCronFreshness = "custom.googleapis.com/cronjob/seconds_since_last_success"
+)
+
+func main() {
+	project := os.Getenv("GOOGLE_CLOUD_PROJECT")
+	if project == "" {
+		logger.Fatal("GOOGLE_CLOUD_PROJECT must be set")
+	}
+	ctx := context.Background()
+	cl, err := monitoring.NewMetricClient(ctx)
+	if err != nil {
+		logger.Fatal("failed to create monitoring client", slog.Any("err", err))
+	}
+	defer cl.Close()
+
+	// cronjob seconds since last success
+	crons := []string{"exporter"}
+	for _, cron := range crons {
+		timeSince, err := getCronFreshness(ctx, cl, project, cron)
+		if err != nil {
+			logger.Fatal("error getting freshness", slog.String("cronjob", cron), slog.Any("err", err))
+		}
+		if err := writeCronFreshness(ctx, cl, project, cron, timeSince); err != nil {
+			logger.Fatal("error writing freshness", slog.String("cronjob", cron), slog.Any("err", err))
+		}
+	}
+}
+
+func getCronFreshness(ctx context.Context, cl *monitoring.MetricClient, project string, cronjob string) (int64, error) {
+	const lookbackDuration = 24 * time.Hour
+	now := time.Now()
+	req := &monitoringpb.ListTimeSeriesRequest{
+		Name:     "projects/" + project,
+		Filter:   fmt.Sprintf("metric.type = %q AND metric.labels.cronjob = %q", kubeStateMetricLastRun, cronjob),
+		Interval: &monitoringpb.TimeInterval{EndTime: timestamppb.New(now), StartTime: timestamppb.New(now.Add(-lookbackDuration))},
+	}
+	it := cl.ListTimeSeries(ctx, req)
+	ts, err := it.Next()
+	if errors.Is(err, iterator.Done) {
+		logger.Warn("last_successful_time was not found for past day", slog.String("cronjob", cronjob))
+		return int64(lookbackDuration / time.Second), nil
+	} else if err != nil {
+		return 0, err
+	}
+	points := ts.GetPoints()
+	if len(points) == 0 { // I'm pretty sure iterator.Done would be returned instead.
+		logger.Warn("time series has no points", slog.String("cronjob", cronjob))
+		return int64(lookbackDuration / time.Second), nil
+	}
+	val := points[0].GetValue().GetDoubleValue()
+
+	return time.Now().Unix() - int64(val), nil
+}
+
+func writeCronFreshness(ctx context.Context, cl *monitoring.MetricClient, project string, cronjob string, value int64) error {
+	now := timestamppb.Now()
+	req := &monitoringpb.CreateTimeSeriesRequest{
+		Name: "projects/" + project,
+		TimeSeries: []*monitoringpb.TimeSeries{{
+			Metric: &metric.Metric{
+				Type: customMetricCronFreshness,
+				Labels: map[string]string{
+					"cronjob": cronjob,
+				},
+			},
+			Resource: &monitoredres.MonitoredResource{
+				Type: "k8s_cluster",
+				Labels: map[string]string{
+					"project_id": project,
+					// hard-coded cluster not ideal, but this is the only cluster in the instance.
+					"location":     "us-central1-f",
+					"cluster_name": "workers",
+				},
+			},
+			Points: []*monitoringpb.Point{{
+				Interval: &monitoringpb.TimeInterval{
+					StartTime: now,
+					EndTime:   now,
+				},
+				Value: &monitoringpb.TypedValue{
+					Value: &monitoringpb.TypedValue_Int64Value{
+						Int64Value: value,
+					},
+				},
+			}},
+		}},
+	}
+
+	return cl.CreateTimeSeries(ctx, req)
+}
diff --git a/go/go.mod b/go/go.mod
@@ -4,11 +4,13 @@ go 1.25.3
 
 require (
 	cloud.google.com/go/datastore v1.21.0
+	cloud.google.com/go/monitoring v1.24.2
 	cloud.google.com/go/pubsub/v2 v2.3.0
 	cloud.google.com/go/storage v1.57.0
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/ossf/osv-schema/bindings/go v0.0.0-20251023235818-d5eaee79a3a5
 	google.golang.org/api v0.254.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c
 	google.golang.org/protobuf v1.36.10
 )
 
@@ -19,7 +21,6 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/iam v1.5.2 // indirect
-	cloud.google.com/go/monitoring v1.24.2 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
@@ -67,7 +68,6 @@ require (
 	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
 	google.golang.org/grpc v1.76.0 // indirect
 )
