[Feature] [Platform] Storage Debug

Author: ajanikow

README.md

@@ -204,7 +204,7 @@ Flags:
       --kubernetes.qps float32                                 Number of queries per second for k8s API (default 32)
       --leader.label.skip                                      Skips Leader Label for the Pod
       --log.format string                                      Set log format. Allowed values: 'pretty', 'JSON'. If empty, default format is used (default "pretty")
-      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, generic-parent-operator, helm, http, inspector, integration-authn-v1, integration-config-v1, integration-envoy-auth-v3, integration-envoy-auth-v3-impl-auth-bearer, integration-envoy-auth-v3-impl-auth-cookie, integration-envoy-auth-v3-impl-custom-openid, integration-envoy-auth-v3-impl-pass-mode, integration-meta-v1, integration-scheduler-v2, integration-storage-v1-s3, integration-storage-v2, integrations, k8s-client, kubernetes, kubernetes-access, kubernetes-client, kubernetes-informer, monitor, networking-route-operator, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, platform-chart-operator, platform-pod-shutdown, platform-service-operator, platform-storage-operator, pod_compare, root, root-event-recorder, scheduler-batchjob-operator, scheduler-cronjob-operator, scheduler-deployment-operator, scheduler-pod-operator, scheduler-profile-operator, server, server-authentication, webhook (default [info])
+      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, generic-parent-operator, helm, http, inspector, integration-authn-v1, integration-config-v1, integration-envoy-auth-v3, integration-envoy-auth-v3-impl-auth-bearer, integration-envoy-auth-v3-impl-auth-cookie, integration-envoy-auth-v3-impl-custom-openid, integration-envoy-auth-v3-impl-pass-mode, integration-meta-v1, integration-scheduler-v2, integration-shutdown-v1, integration-storage-v1-s3, integration-storage-v2, integrations, k8s-client, kubernetes, kubernetes-access, kubernetes-client, kubernetes-informer, monitor, networking-route-operator, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, platform-chart-operator, platform-pod-shutdown, platform-service-operator, platform-storage-operator, pod_compare, root, root-event-recorder, scheduler-batchjob-operator, scheduler-cronjob-operator, scheduler-deployment-operator, scheduler-pod-operator, scheduler-profile-operator, server, server-authentication, webhook (default [info])
       --log.sampling                                           If true, operator will try to minimize duplication of logging events (default true)
       --log.stdout                                             If true, operator will log to the stdout (default true)
       --memory-limit uint                                      Define memory limit for hard shutdown and the dump of goroutines. Used for testing

docs/cli/arangodb_operator.md

@@ -86,7 +86,7 @@ Flags:
       --kubernetes.qps float32                                 Number of queries per second for k8s API (default 32)
       --leader.label.skip                                      Skips Leader Label for the Pod
       --log.format string                                      Set log format. Allowed values: 'pretty', 'JSON'. If empty, default format is used (default "pretty")
-      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, generic-parent-operator, helm, http, inspector, integration-authn-v1, integration-config-v1, integration-envoy-auth-v3, integration-envoy-auth-v3-impl-auth-bearer, integration-envoy-auth-v3-impl-auth-cookie, integration-envoy-auth-v3-impl-custom-openid, integration-envoy-auth-v3-impl-pass-mode, integration-meta-v1, integration-scheduler-v2, integration-storage-v1-s3, integration-storage-v2, integrations, k8s-client, kubernetes, kubernetes-access, kubernetes-client, kubernetes-informer, monitor, networking-route-operator, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, platform-chart-operator, platform-pod-shutdown, platform-service-operator, platform-storage-operator, pod_compare, root, root-event-recorder, scheduler-batchjob-operator, scheduler-cronjob-operator, scheduler-deployment-operator, scheduler-pod-operator, scheduler-profile-operator, server, server-authentication, webhook (default [info])
+      --log.level stringArray                                  Set log levels in format <level> or <logger>=<level>. Possible loggers: action, agency, api-server, assertion, backup-operator, chaos-monkey, crd, deployment, deployment-ci, deployment-reconcile, deployment-replication, deployment-resilience, deployment-resources, deployment-storage, deployment-storage-pc, deployment-storage-service, generic-parent-operator, helm, http, inspector, integration-authn-v1, integration-config-v1, integration-envoy-auth-v3, integration-envoy-auth-v3-impl-auth-bearer, integration-envoy-auth-v3-impl-auth-cookie, integration-envoy-auth-v3-impl-custom-openid, integration-envoy-auth-v3-impl-pass-mode, integration-meta-v1, integration-scheduler-v2, integration-shutdown-v1, integration-storage-v1-s3, integration-storage-v2, integrations, k8s-client, kubernetes, kubernetes-access, kubernetes-client, kubernetes-informer, monitor, networking-route-operator, operator, operator-arangojob-handler, operator-v2, operator-v2-event, operator-v2-worker, panics, platform-chart-operator, platform-pod-shutdown, platform-service-operator, platform-storage-operator, pod_compare, root, root-event-recorder, scheduler-batchjob-operator, scheduler-cronjob-operator, scheduler-deployment-operator, scheduler-pod-operator, scheduler-profile-operator, server, server-authentication, webhook (default [info])
       --log.sampling                                           If true, operator will try to minimize duplication of logging events (default true)
       --log.stdout                                             If true, operator will log to the stdout (default true)
       --memory-limit uint                                      Define memory limit for hard shutdown and the dump of goroutines. Used for testing

docs/cli/arangodb_operator_integration.md

@@ -74,6 +74,9 @@ Flags:
       --integration.scheduler.v2.internal                           Defines if Internal access to service scheduler.v2 is enabled (Env: INTEGRATION_SCHEDULER_V2_INTERNAL) (default true)
       --integration.scheduler.v2.namespace string                   Kubernetes Namespace (Env: INTEGRATION_SCHEDULER_V2_NAMESPACE) (default "default")
       --integration.shutdown.v1                                     ShutdownV1 Handler (Env: INTEGRATION_SHUTDOWN_V1)
+      --integration.shutdown.v1.debug.enabled                       Defines if debug extension is enabled (Env: INTEGRATION_SHUTDOWN_V1_DEBUG_ENABLED)
+      --integration.shutdown.v1.debug.path string                   Path of the Debug Directory (Env: INTEGRATION_SHUTDOWN_V1_DEBUG_PATH) (default "/debug")
+      --integration.shutdown.v1.debug.timeout duration              Timeout of the Debug action (Env: INTEGRATION_SHUTDOWN_V1_DEBUG_TIMEOUT) (default 1m0s)
       --integration.shutdown.v1.external                            Defines if External access to service shutdown.v1 is enabled (Env: INTEGRATION_SHUTDOWN_V1_EXTERNAL)
       --integration.shutdown.v1.internal                            Defines if Internal access to service shutdown.v1 is enabled (Env: INTEGRATION_SHUTDOWN_V1_INTERNAL) (default true)
       --integration.storage.v2                                      StorageBucket V2 Integration (Env: INTEGRATION_STORAGE_V2)

docs/integration-sidecar.md

@@ -19,6 +19,8 @@ Integration Sidecar is supported in a few resources managed by Operator:
 - ArangoSchedulerCronJob (scheduler.arangodb.com/v1beta1)
 - ArangoSchedulerPod (scheduler.arangodb.com/v1beta1)
 
+S tandard Kubernetes Resources (like Pod) are also supported with Webhook extension is enabled.
+
 To enable integration sidecar for specific deployment label needs to be defined:
 
 ```yaml
@@ -97,6 +99,9 @@ To enable:
 metadata:
   labels:
     integration.profiles.arangodb.com/storage: v1
+    
+    # Optionally, enable DebugPackage integration
+    storage.integration.profiles.arangodb.com/debug: "enabled"
 ```
 
 #### [Storage V2](./integration/storage.v2.md)

docs/integration/shutdown.v1.md

@@ -31,3 +31,15 @@ spec:
 ```
 
 Pod will receive shutdown request on port `port1` if containers `app` and `app2` will be in non running state.
+
+## Extensions
+
+### DebugPackage PreShutdown Hook
+
+Example:
+
+```yaml
+metadata:
+  labels:
+    core.shutdown.arangodb.com/app: "wait"
+```

integrations/shutdown/v1/configuration.go

@@ -0,0 +1,35 @@
+//
+// DISCLAIMER
+//
+// Copyright 2025 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import "time"
+
+type Configuration struct {
+	Debug ConfigurationDebug `json:"debug,omitempty"`
+}
+
+type ConfigurationDebug struct {
+	Enabled bool `json:"enabled,omitempty"`
+
+	Path string `json:"path,omitempty"`
+
+	Timeout time.Duration `json:"timeout,omitempty"`
+}

integrations/shutdown/v1/impl.go

@@ -22,18 +22,33 @@ package v1
 
 import (
 	"context"
+	"io/fs"
+	"path/filepath"
+	goStrings "strings"
 	"time"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"google.golang.org/grpc"
 
 	pbSharedV1 "github.com/arangodb/kube-arangodb/integrations/shared/v1/definition"
 	pbShutdownV1 "github.com/arangodb/kube-arangodb/integrations/shutdown/v1/definition"
+	pbStorageV2 "github.com/arangodb/kube-arangodb/integrations/storage/v2/definition"
+	"github.com/arangodb/kube-arangodb/pkg/util/closer"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	ugrpc "github.com/arangodb/kube-arangodb/pkg/util/grpc"
 	"github.com/arangodb/kube-arangodb/pkg/util/svc"
 )
 
-func New(closer context.CancelFunc) svc.Handler {
-	return &impl{closer: closer}
+func New(cfg Configuration, c context.CancelFunc) svc.Handler {
+	var z = &impl{
+		closer: c,
+		cfg:    cfg,
+	}
+
+	z.close = closer.CloseOnce(z)
+
+	return z
 }
 
 var _ pbShutdownV1.ShutdownV1Server = &impl{}
@@ -42,9 +57,50 @@ var _ svc.Handler = &impl{}
 type impl struct {
 	pbShutdownV1.UnimplementedShutdownV1Server
 
+	cfg Configuration
+
+	close closer.Close
+
 	closer context.CancelFunc
 }
 
+func (i *impl) Close() error {
+	defer i.closer()
+
+	ctx, c := context.WithTimeout(context.Background(), i.cfg.Debug.Timeout)
+	defer c()
+
+	time.Sleep(50 * time.Millisecond)
+
+	if i.cfg.Debug.Enabled {
+		// Need to fetch Debug Details
+		if addr, ok := constants.INTEGRATION_SERVICE_ADDRESS.Lookup(); ok {
+			client, close, err := ugrpc.NewGRPCClient(ctx, pbStorageV2.NewStorageV2Client, addr)
+			if err != nil {
+				return err
+			}
+
+			defer close.Close()
+
+			if err := filepath.Walk(i.cfg.Debug.Path, func(path string, info fs.FileInfo, err error) error {
+				k := goStrings.TrimPrefix(path, i.cfg.Debug.Path)
+
+				if _, err := pbStorageV2.SendFile(ctx, client, k, path); err != nil {
+					logger.Err(err).Warn("Failed to send file to server for DebugPackage")
+				}
+
+				return nil
+			}); err != nil {
+				return err
+			}
+		} else {
+			return errors.Errorf("Address of the Service not defined")
+		}
+	}
+
+	return nil
+}
+
 func (i *impl) Name() string {
 	return pbShutdownV1.Name
 }
@@ -63,10 +119,12 @@ func (i *impl) Gateway(ctx context.Context, mux *runtime.ServeMux) error {
 
 func (i *impl) Shutdown(ctx context.Context, empty *pbSharedV1.Empty) (*pbSharedV1.Empty, error) {
 	go func() {
-		defer i.closer()
-
-		time.Sleep(50 * time.Millisecond)
+		if err := i.close.Close(); err != nil {
+			logger.Err(err).Warn("Shutting down failed")
+		}
 	}()
 
+	logger.Info("Shutting down")
+
 	return &pbSharedV1.Empty{}, nil
 }

integrations/shutdown/v1/impl_test.go

@@ -40,7 +40,7 @@ func Test_ShutdownGRPC(t *testing.T) {
 
 	local, err := svc.NewService(svc.Configuration{
 		Address: "127.0.0.1:0",
-	}, New(c))
+	}, New(Configuration{}, c))
 	require.NoError(t, err)
 
 	start := local.Start(ctx)

integrations/shutdown/v1/logger.go

@@ -0,0 +1,25 @@
+//
+// DISCLAIMER
+//
+// Copyright 2025 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+import "github.com/arangodb/kube-arangodb/pkg/logging"
+
+var logger = logging.Global().RegisterAndGetLogger("integration-shutdown-v1", logging.Info)

integrations/shutdown/v1/service_test.go

@@ -37,7 +37,7 @@ import (
 func Client(t *testing.T, ctx context.Context, c context.CancelFunc) pbShutdownV1.ShutdownV1Client {
 	local, err := svc.NewService(svc.Configuration{
 		Address: "127.0.0.1:0",
-	}, New(c))
+	}, New(Configuration{}, c))
 	require.NoError(t, err)
 
 	start := local.Start(ctx)

integrations/storage/v2/definition/helpers.go

@@ -23,6 +23,7 @@ package definition
 import (
 	"context"
 	"io"
+	"os"
 
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
@@ -31,6 +32,24 @@ import (
 
 const BufferSize = 4094
 
+func SendFile(ctx context.Context, client StorageV2Client, key string, path string) (*StorageV2WriteObjectResponse, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	p, err := Send(ctx, client, key, f)
+	if err != nil {
+		if cerr := f.Close(); cerr != nil {
+			return nil, cerr
+		}
+
+		return nil, err
+	}
+
+	return p, nil
+}
+
 func Send(ctx context.Context, client StorageV2Client, key string, in io.Reader) (*StorageV2WriteObjectResponse, error) {
 	cache := make([]byte, BufferSize)
 

pkg/deployment/resources/arango_profiles.go

@@ -60,14 +60,24 @@ var (
 	inspectArangoProfilesDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_arango_profiles_duration", "Amount of time taken by a single inspection of all ArangoProfiles for a deployment (in sec)", metrics.DeploymentName)
 )
 
-func matchArangoProfilesLabels(labels map[string]string) *schedulerApi.ProfileSelectors {
+func matchArangoProfilesLabels(labels map[string]string, additional ...util.KV[string, string]) *schedulerApi.ProfileSelectors {
 	if labels == nil {
 		return nil
 	}
 
+	out := make(map[string]string)
+
+	for k, v := range labels {
+		out[k] = v
+	}
+
+	for _, k := range additional {
+		out[k.K] = k.V
+	}
+
 	return &schedulerApi.ProfileSelectors{
 		Label: &meta.LabelSelector{
-			MatchLabels: labels,
+			MatchLabels: out,
 		},
 	}
 }
@@ -84,7 +94,7 @@ func (r *Resources) EnsureArangoProfiles(ctx context.Context, cachedStatus inspe
 
 	reconcileRequired := k8sutil.NewReconcile(cachedStatus)
 
-	gen := func(name, version string, generator func() (integrationsSidecar.Integration, bool)) func() (string, *schedulerApi.ArangoProfile, error) {
+	gen := func(name, version string, generator func() (integrationsSidecar.Integration, bool), selectors ...util.KV[string, string]) func() (string, *schedulerApi.ArangoProfile, error) {
 		return func() (string, *schedulerApi.ArangoProfile, error) {
 			counterMetric.Inc()
 			fullName := fmt.Sprintf("%s-int-%s-%s", deploymentName, name, version)
@@ -113,7 +123,7 @@ func (r *Resources) EnsureArangoProfiles(ctx context.Context, cachedStatus inspe
 					Selectors: matchArangoProfilesLabels(map[string]string{
 						constants.ProfilesDeployment: deploymentName,
 						key:                          v,
-					}),
+					}, selectors...),
 					Template: integration,
 				},
 			}, nil
@@ -170,6 +180,10 @@ func (r *Resources) EnsureArangoProfiles(ctx context.Context, cachedStatus inspe
 			DeploymentName: apiObject.GetName(),
 		})),
 		gen(constants.ProfilesIntegrationShutdown, constants.ProfilesIntegrationV1, always(integrationsSidecar.IntegrationShutdownV1{})),
+		gen(constants.ProfilesIntegrationShutdown, constants.ProfilesIntegrationV1, always(integrationsSidecar.ExtensionShutdownV1Debug{}), util.KV[string, string]{
+			K: constants.ProfilesExtensionStorageDebug,
+			V: constants.ProfilesExtensionEnabled,
+		}),
 		gen(constants.ProfilesIntegrationEnvoy, constants.ProfilesIntegrationV3, always(integrationsSidecar.IntegrationEnvoyV3{Spec: spec})),
 		gen(constants.ProfilesIntegrationStorage, constants.ProfilesIntegrationV1, func() (integrationsSidecar.Integration, bool) {
 			if v, err := cachedStatus.ArangoPlatformStorage().V1Alpha1(); err == nil {

pkg/integrations/register.go

@@ -281,7 +281,7 @@ func (c *configuration) runWithContext(ctx context.Context, cancel context.Cance
 	healthHandlers = append(healthHandlers, pong)
 
 	if c.health.shutdownEnabled {
-		healthHandlers = append(healthHandlers, pbImplShutdownV1.New(cancel))
+		healthHandlers = append(healthHandlers, pbImplShutdownV1.New(pbImplShutdownV1.Configuration{}, cancel))
 	}
 
 	health, err := svc.NewHealthService(healthConfig, svc.Readiness, healthHandlers...)

pkg/integrations/shutdown_v1.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2024 ArangoDB GmbH, Cologne, Germany
+// Copyright 2024-2025 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -22,11 +22,13 @@ package integrations
 
 import (
 	"context"
+	"time"
 
 	"github.com/spf13/cobra"
 
 	pbImplShutdownV1 "github.com/arangodb/kube-arangodb/integrations/shutdown/v1"
 	pbShutdownV1 "github.com/arangodb/kube-arangodb/integrations/shutdown/v1/definition"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 	"github.com/arangodb/kube-arangodb/pkg/util/shutdown"
 	"github.com/arangodb/kube-arangodb/pkg/util/svc"
 )
@@ -38,10 +40,11 @@ func init() {
 }
 
 type shutdownV1 struct {
+	Configuration pbImplShutdownV1.Configuration
 }
 
 func (s *shutdownV1) Handler(ctx context.Context, cmd *cobra.Command) (svc.Handler, error) {
-	return pbImplShutdownV1.New(shutdown.Stop), nil
+	return pbImplShutdownV1.New(s.Configuration, shutdown.Stop), nil
 }
 
 func (s *shutdownV1) Name() string {
@@ -53,7 +56,11 @@ func (s *shutdownV1) Description() string {
 }
 
 func (s *shutdownV1) Register(cmd *cobra.Command, fs FlagEnvHandler) error {
-	return nil
+	return errors.Errors(
+		fs.BoolVar(&s.Configuration.Debug.Enabled, "debug.enabled", false, "Defines if debug extension is enabled"),
+		fs.StringVar(&s.Configuration.Debug.Path, "debug.path", "/debug", "Path of the Debug Directory"),
+		fs.DurationVar(&s.Configuration.Debug.Timeout, "debug.timeout", time.Minute, "Timeout of the Debug action"),
+	)
 }
 
 func (*shutdownV1) Init(ctx context.Context, cmd *cobra.Command) error {