WIP: e2e test fixes

Author: micahhausler

Makefile

@@ -401,7 +401,6 @@ e2e-deploy-manifest:
 	kubectl apply -f manifest_staging/deploy/rbac-secretproviderclass.yaml
 	kubectl apply -f manifest_staging/deploy/rbac-secretproviderrotation.yaml
 	kubectl apply -f manifest_staging/deploy/rbac-secretprovidersyncing.yaml
-	kubectl apply -f manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
 	kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasses.yaml
 	kubectl apply -f manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml
 	kubectl apply -f manifest_staging/deploy/role-secretproviderclasses-admin.yaml
@@ -429,6 +428,7 @@ e2e-helm-deploy:
 		--set syncSecret.enabled=true \
 		--set enableSecretRotation=true \
 		--set rotationPollInterval=30s \
+		--set requiresRepublish=true \
 		--set tokenRequests[0].audience="aud1" \
 		--set tokenRequests[1].audience="aud2" \
 		--set tokenRequests[2].audience="conjur" \
@@ -533,16 +533,6 @@ manifests: $(CONTROLLER_GEN) $(KUSTOMIZE)  ## Generate manifests e.g. CRD, RBAC
 	@sed -i '1s/^/{{ if .Values.enableSecretRotation }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation_binding.yaml
 	@sed -i '/^roleRef:/i \ \ labels:\n{{ include \"sscd.labels\" . | indent 4 }}' manifest_staging/charts/secrets-store-csi-driver/templates/role-rotation_binding.yaml
 
-	# Generate token requests specific RBAC
-	$(CONTROLLER_GEN) rbac:roleName=secretprovidertokenrequest-role paths="./controllers/tokenrequest" output:dir=config/rbac-tokenrequest
-	$(KUSTOMIZE) build config/rbac-tokenrequest -o manifest_staging/deploy/rbac-secretprovidertokenrequest.yaml
-	cp config/rbac-tokenrequest/role.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
-	cp config/rbac-tokenrequest/role_binding.yaml manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
-	@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
-	@sed -i '/^rules:/i \ \ labels:\n{{ include \"sscd.labels\" . | indent 4 }}' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest.yaml
-	@sed -i '1s/^/{{ if .Values.tokenRequests }}\n/gm; s/namespace: .*/namespace: {{ .Release.Namespace }}/gm; $$s/$$/\n{{ end }}/gm' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
-	@sed -i '/^roleRef:/i \ \ labels:\n{{ include \"sscd.labels\" . | indent 4 }}' manifest_staging/charts/secrets-store-csi-driver/templates/role-tokenrequest_binding.yaml
-
 .PHONY: generate-protobuf
 generate-protobuf: $(PROTOC) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) # generates protobuf
 	@PATH=$(PATH):$(TOOLS_BIN_DIR) $(PROTOC) -I . provider/v1alpha1/service.proto --go-grpc_out=require_unimplemented_servers=false:. --go_out=.

config/rbac-tokenrequest/kustomization.yaml

@@ -50,10 +50,6 @@ kubectl apply -f deploy/rbac-secretprovidersyncing.yaml
 # required to enable this feature
 kubectl apply -f deploy/rbac-secretproviderrotation.yaml
 
-# If using the CSI Driver token requests feature (https://kubernetes-csi.github.io/docs/token-requests.html) to use
-# pod/workload identity to request a token and use with providers
-kubectl apply -f deploy/rbac-secretprovidertokenrequest.yaml
-
 # [OPTIONAL] To deploy driver on windows nodes
 kubectl apply -f deploy/secrets-store-csi-driver-windows.yaml
 ```

config/rbac-tokenrequest/role.yaml

@@ -40,14 +40,13 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
 
 @test "setup mock provider validation config" {
   if [[ -n "${VALIDATE_TOKENS_AUDIENCE}" ]]; then
-    # configure the mock provider to validate the token requests
     kubectl create ns enable-token-requests
-    local curl_pod_name=curl-$(openssl rand -hex 5)
-    kubectl run ${curl_pod_name} -n enable-token-requests --image=curlimages/curl:7.75.0 --labels="util=enable-token-requests" -- tail -f /dev/null
-    kubectl wait -n enable-token-requests --for=condition=Ready --timeout=60s pod ${curl_pod_name}
-    local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}")
-    run kubectl exec ${curl_pod_name} -n enable-token-requests -- curl http://${pod_ip}:8080/validate-token-requests?audience=${VALIDATE_TOKENS_AUDIENCE}
-    kubectl delete pod -l util=enable-token-requests -n enable-token-requests --force --grace-period 0
+    kubectl apply -f $BATS_TESTS_DIR/demo-deployment-e2e-provider.yaml
+    local pod_name=$(kubectl get pod -n enable-token-requests -l app.kubernetes.io/name=demo-pod -o jsonpath="{.items[0].metadata.name}")
+    kubectl wait -n enable-token-requests --for=condition=Ready --timeout=60s pod ${pod_name}
+    run kubectl exec ${pod_name} -n enable-token-requests -- cat /mnt/e2e/tokens.json
+    kubectl delete -f $BATS_TESTS_DIR/demo-deployment-e2e-provider.yaml --timeout=5s
+    kubectl delete pod -l app.kubernetes.io/name=demo-pod -n enable-token-requests --force --grace-period 0
     kubectl delete ns enable-token-requests
   fi
 
@@ -98,14 +97,6 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
   run kubectl get clusterrolebinding/secretprovidersyncing-rolebinding
   assert_success
 
-  # validate token request role and rolebinding only when token requests are set
-  if [[ -n "${VALIDATE_TOKENS_AUDIENCE}" ]]; then
-    run kubectl get clusterrole/secretprovidertokenrequest-role
-    assert_success
-
-    run kubectl get clusterrolebinding/secretprovidertokenrequest-rolebinding
-    assert_success
-  fi
 }
 
 @test "[v1alpha1] deploy e2e-provider secretproviderclass crd" {

config/rbac-tokenrequest/role_binding.yaml

@@ -0,0 +1,44 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: empty-e2e-provider
+  namespace: enable-token-requests
+spec:
+  provider: e2e-provider
+  parameters:
+    objects: |
+      array:[]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: demo-pod
+  namespace: enable-token-requests
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: demo-pod
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: demo-pod
+    spec:
+      containers:
+      - image: registry.k8s.io/e2e-test-images/busybox:1.29-4
+        command:
+        - /bin/sleep
+        - "10000"
+        imagePullPolicy: Always
+        name: demo
+        volumeMounts:
+        - name: secrets-store-inline
+          mountPath: /mnt/e2e
+          readOnly: true
+      volumes:
+      - name: secrets-store-inline
+        csi:
+          driver: secrets-store.csi.k8s.io
+          readOnly: true
+          volumeAttributes:
+            secretProviderClass: "empty-e2e-provider"

deploy/rbac-secretprovidertokenrequest.yaml

@@ -77,7 +77,6 @@ func mainErr() error {
 		os.Setenv("ROTATION_ENABLED", "false")
 
 		http.HandleFunc("/rotation", server.RotationHandler)
-		http.HandleFunc("/validate-token-requests", server.ValidateTokenAudienceHandler)
 
 		server := &http.Server{
 			Addr:              ":8080",

docs/book/src/getting-started/installation.md

@@ -181,23 +181,6 @@ func (s *Server) Mount(ctx context.Context, req *v1alpha1.MountRequest) (*v1alph
 		})
 	}
 
-	// if validate token flag is set, we want to check the service account tokens as passed
-	// as part of the mount attributes.
-	// In case of 1.21+, kubelet will generate the token and pass it as part of the volume context.
-	// The driver will pass this to the provider as part of the mount request.
-	// For 1.20, the driver will generate the token and pass it to the provider as part of the mount request.
-	// Irrespective of the kubernetes version, the rotation handler in the driver will generate the token
-	// and pass it to the provider as part of the mount request.
-	// VALIDATE_TOKENS_AUDIENCE environment variable will be a comma separated list of audiences configured in the csidriver object
-	// If this env var is not set, this could mean we are running an older version of driver.
-	tokenAudiences := os.Getenv("VALIDATE_TOKENS_AUDIENCE")
-	klog.InfoS("tokenAudiences", "tokenAudiences", tokenAudiences)
-	if tokenAudiences != "" {
-		if err := validateTokens(tokenAudiences, attrib[serviceAccountTokensAttribute]); err != nil {
-			return nil, fmt.Errorf("failed to validate token, error: %w", err)
-		}
-	}
-
 	m.Lock()
 	podCache[attrib[podUIDAttribute]] = true
 	m.Unlock()
@@ -249,32 +232,3 @@ func RotationHandler(w http.ResponseWriter, r *http.Request) {
 	os.Setenv("ROTATION_ENABLED", r.FormValue("rotated"))
 	klog.InfoS("Rotation response enabled")
 }
-
-// ValidateTokenAudienceHandler enables token validation for the mock provider
-// This is only required because older version of the driver don't generate a token
-// TODO(aramase): remove this after the supported driver releases are v1.1.0+
-func ValidateTokenAudienceHandler(w http.ResponseWriter, r *http.Request) {
-	// enable rotation response
-	os.Setenv("VALIDATE_TOKENS_AUDIENCE", r.FormValue("audience"))
-	klog.InfoS("Validation for token requests audience", "audience", os.Getenv("VALIDATE_TOKENS_AUDIENCE"))
-}
-
-// validateTokens checks there are tokens for distinct audiences in the
-// service account token attribute.
-func validateTokens(tokenAudiences, saTokens string) error {
-	ta := strings.Split(strings.TrimSpace(tokenAudiences), ",")
-	if saTokens == "" {
-		return fmt.Errorf("service account tokens is not set")
-	}
-	tokens := make(map[string]interface{})
-	if err := json.Unmarshal([]byte(saTokens), &tokens); err != nil {
-		return fmt.Errorf("failed to unmarshal service account tokens, error: %w", err)
-	}
-	for _, a := range ta {
-		if _, ok := tokens[a]; !ok {
-			return fmt.Errorf("service account token for audience %s is not set", a)
-		}
-		klog.InfoS("Validated service account token", "audience", a)
-	}
-	return nil
-}