feat: adding volume mounts

README.md

@@ -25,6 +25,8 @@ This service allows for creating and managing isolated user environments (sandbo
 - Go 1.21+
 - `kubectl` configured with appropriate permissions
 
+For information about volume permissions and how they're handled, see the [Volume Permissions Guide](docs/volume-permissions.md).
+
 ## Installation
 
 ### Clone the repository

docs/volume-permissions.md

@@ -0,0 +1,88 @@
+# Volume Permissions in Kubernetes
+
+This document explains how persistent volume permissions are handled in the k8sgo application.
+
+## Overview
+
+The k8sgo application creates Kubernetes pods that run as `nodeuser` (UID 1001, GID 1001) and need to write to persistent volumes mounted at `/home/nodeuser/.iris`. By default, Kubernetes volume mounts often have permissions that prevent non-root users from writing to them.
+
+## Solution Implementation
+
+We've implemented two key strategies to ensure proper permissions:
+
+### 1. PodSecurityContext
+
+The pod specification includes a SecurityContext with:
+
+```yaml
+securityContext:
+  fsGroup: 1001  # nodeuser's group ID
+  runAsUser: 1001  # nodeuser's user ID
+  runAsGroup: 1001
+  fsGroupChangePolicy: OnRootMismatch  # More efficient permission changes
+```
+
+The `fsGroup` setting ensures the volume is accessible to the group, while `fsGroupChangePolicy: OnRootMismatch` makes permission changes more efficient by only applying them when necessary.
+
+### 2. Init Container
+
+An init container runs before the main application container to explicitly set permissions:
+
+```yaml
+initContainers:
+- name: volume-permissions
+  image: busybox:latest
+  command:
+  - sh
+  - -c
+  - mkdir -p /home/nodeuser/.iris && chown -R 1001:1001 /home/nodeuser/.iris
+  volumeMounts:
+  - name: user-data
+    mountPath: /home/nodeuser/.iris
+```
+
+This init container ensures that:
+1. The `.iris` directory exists
+2. It has the correct ownership (nodeuser:nodeuser)
+
+## Testing
+
+When testing new deployments, you can verify permissions are correct by:
+
+1. Connecting to the pod:
+   ```bash
+   kubectl exec -it <pod-name> -n user-sandboxes -- /bin/bash
+   ```
+
+2. Checking permissions:
+   ```bash
+   ls -la /home/nodeuser/.iris
+   ```
+
+3. Testing write access:
+   ```bash
+   sudo -u nodeuser touch /home/nodeuser/.iris/test.txt
+   ```
+
+## Troubleshooting
+
+If permission issues persist:
+
+1. Verify the UID/GID:
+   ```bash
+   kubectl exec -n user-sandboxes <pod-name> -- id nodeuser
+   ```
+
+2. Check volume mount permissions:
+   ```bash
+   kubectl exec -n user-sandboxes <pod-name> -- ls -la /home/nodeuser
+   ```
+
+3. Inspect the pod's security context:
+   ```bash
+   kubectl get pod <pod-name> -n user-sandboxes -o yaml | grep -A15 securityContext
+   ```
+
+## Storage Class Considerations
+
+Different storage classes may handle permissions differently. If you're using a storage class that doesn't respect `fsGroup`, you may need to rely more heavily on the init container approach.

fix-permissions-pod.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: fix-permissions-test
+  namespace: user-sandboxes
+spec:
+  securityContext:
+    fsGroup: 1001
+    runAsUser: 1001
+    runAsGroup: 1001
+  containers:
+  - name: test
+    image: busybox:latest
+    command:
+    - sh
+    - -c
+    - |
+      mkdir -p /home/nodeuser/.iris
+      echo "test" > /home/nodeuser/.iris/test.txt
+      ls -la /home/nodeuser/.iris
+      cat /home/nodeuser/.iris/test.txt
+      echo "Test complete"
+      sleep 3600
+    volumeMounts:
+    - name: user-data
+      mountPath: /home/nodeuser/.iris
+  volumes:
+  - name: user-data
+    persistentVolumeClaim:
+      claimName: shanurrhaman-pvc

internal/k8s/client.go

@@ -19,6 +19,19 @@ import (
 	"k8s.io/client-go/util/homedir"
 )
 
+// Helper function to convert int64 to pointer
+func pointer(i int64) *int64 {
+	return &i
+}
+
+// Constants for user and group IDs
+const (
+	// NodeUserID is the user ID for nodeuser in the container
+	NodeUserID int64 = 1001
+	// NodeGroupID is the group ID for nodeuser in the container
+	NodeGroupID int64 = 1001
+)
+
 // Client is a Kubernetes client wrapper
 type Client struct {
 	clientset *kubernetes.Clientset
@@ -225,6 +238,8 @@ func (c *Client) createNodeEnvConfigMap(ctx context.Context, userID string, node
 
 // createDeployment creates a deployment for the user's sandbox
 func (c *Client) createDeployment(ctx context.Context, userID string, envVars map[string]string, hasNodeEnv bool) error {
+	// Add debug log to confirm security context is being applied
+	log.Printf("Creating deployment with securityContext (fsGroup: %d, runAsUser: %d)", NodeGroupID, NodeUserID)
 	deploymentName := fmt.Sprintf("%s-deployment", userID)
 
 	// Create deployment
@@ -305,6 +320,32 @@ func (c *Client) createDeployment(ctx context.Context, userID string, envVars ma
 					},
 				},
 				Spec: corev1.PodSpec{
+					SecurityContext: &corev1.PodSecurityContext{
+						FSGroup:            pointer(NodeGroupID),
+						RunAsUser:          pointer(NodeUserID),
+						RunAsGroup:         pointer(NodeGroupID),
+						FSGroupChangePolicy: func() *corev1.PodFSGroupChangePolicy {
+							policy := corev1.FSGroupChangeOnRootMismatch
+							return &policy
+						}(),
+					},
+					InitContainers: []corev1.Container{
+						{
+							Name:  "volume-permissions",
+							Image: "busybox:latest",
+							Command: []string{
+								"sh",
+								"-c",
+								fmt.Sprintf("mkdir -p /home/nodeuser/.iris && chown -R %d:%d /home/nodeuser/.iris", NodeUserID, NodeGroupID),
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "user-data",
+									MountPath: "/home/nodeuser/.iris",
+								},
+							},
+						},
+					},
 					Containers: []corev1.Container{
 						{
 							Name:  "sandbox",