diff --git a/README.md b/README.md index 1569a68..188412c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ This service allows for creating and managing isolated user environments (sandbo - Go 1.21+ - `kubectl` configured with appropriate permissions +For information about volume permissions and how they're handled, see the [Volume Permissions Guide](docs/volume-permissions.md). + ## Installation ### Clone the repository diff --git a/docs/volume-permissions.md b/docs/volume-permissions.md new file mode 100644 index 0000000..06b89cb --- /dev/null +++ b/docs/volume-permissions.md @@ -0,0 +1,88 @@ +# Volume Permissions in Kubernetes + +This document explains how persistent volume permissions are handled in the k8sgo application. + +## Overview + +The k8sgo application creates Kubernetes pods that run as `nodeuser` (UID 1001, GID 1001) and need to write to persistent volumes mounted at `/home/nodeuser/.iris`. By default, Kubernetes volume mounts often have permissions that prevent non-root users from writing to them. + +## Solution Implementation + +We've implemented two key strategies to ensure proper permissions: + +### 1. PodSecurityContext + +The pod specification includes a SecurityContext with: + +```yaml +securityContext: + fsGroup: 1001 # nodeuser's group ID + runAsUser: 1001 # nodeuser's user ID + runAsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch # More efficient permission changes +``` + +The `fsGroup` setting ensures the volume is accessible to the group, while `fsGroupChangePolicy: OnRootMismatch` makes permission changes more efficient by only applying them when necessary. + +### 2. Init Container + +An init container runs before the main application container to explicitly set permissions: + +```yaml +initContainers: +- name: volume-permissions + image: busybox:latest + command: + - sh + - -c + - mkdir -p /home/nodeuser/.iris && chown -R 1001:1001 /home/nodeuser/.iris + volumeMounts: + - name: user-data + mountPath: /home/nodeuser/.iris +``` + +This init container ensures that: +1. The `.iris` directory exists +2. It has the correct ownership (nodeuser:nodeuser) + +## Testing + +When testing new deployments, you can verify permissions are correct by: + +1. Connecting to the pod: + ```bash + kubectl exec -it -n user-sandboxes -- /bin/bash + ``` + +2. Checking permissions: + ```bash + ls -la /home/nodeuser/.iris + ``` + +3. Testing write access: + ```bash + sudo -u nodeuser touch /home/nodeuser/.iris/test.txt + ``` + +## Troubleshooting + +If permission issues persist: + +1. Verify the UID/GID: + ```bash + kubectl exec -n user-sandboxes -- id nodeuser + ``` + +2. Check volume mount permissions: + ```bash + kubectl exec -n user-sandboxes -- ls -la /home/nodeuser + ``` + +3. Inspect the pod's security context: + ```bash + kubectl get pod -n user-sandboxes -o yaml | grep -A15 securityContext + ``` + +## Storage Class Considerations + +Different storage classes may handle permissions differently. If you're using a storage class that doesn't respect `fsGroup`, you may need to rely more heavily on the init container approach. \ No newline at end of file diff --git a/fix-permissions-pod.yaml b/fix-permissions-pod.yaml new file mode 100644 index 0000000..3d7e20d --- /dev/null +++ b/fix-permissions-pod.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Pod +metadata: + name: fix-permissions-test + namespace: user-sandboxes +spec: + securityContext: + fsGroup: 1001 + runAsUser: 1001 + runAsGroup: 1001 + containers: + - name: test + image: busybox:latest + command: + - sh + - -c + - | + mkdir -p /home/nodeuser/.iris + echo "test" > /home/nodeuser/.iris/test.txt + ls -la /home/nodeuser/.iris + cat /home/nodeuser/.iris/test.txt + echo "Test complete" + sleep 3600 + volumeMounts: + - name: user-data + mountPath: /home/nodeuser/.iris + volumes: + - name: user-data + persistentVolumeClaim: + claimName: shanurrhaman-pvc \ No newline at end of file diff --git a/internal/k8s/client.go b/internal/k8s/client.go index 5f05012..3a55e7c 100644 --- a/internal/k8s/client.go +++ b/internal/k8s/client.go @@ -19,6 +19,19 @@ import ( "k8s.io/client-go/util/homedir" ) +// Helper function to convert int64 to pointer +func pointer(i int64) *int64 { + return &i +} + +// Constants for user and group IDs +const ( + // NodeUserID is the user ID for nodeuser in the container + NodeUserID int64 = 1001 + // NodeGroupID is the group ID for nodeuser in the container + NodeGroupID int64 = 1001 +) + // Client is a Kubernetes client wrapper type Client struct { clientset *kubernetes.Clientset @@ -225,6 +238,8 @@ func (c *Client) createNodeEnvConfigMap(ctx context.Context, userID string, node // createDeployment creates a deployment for the user's sandbox func (c *Client) createDeployment(ctx context.Context, userID string, envVars map[string]string, hasNodeEnv bool) error { + // Add debug log to confirm security context is being applied + log.Printf("Creating deployment with securityContext (fsGroup: %d, runAsUser: %d)", NodeGroupID, NodeUserID) deploymentName := fmt.Sprintf("%s-deployment", userID) // Create deployment @@ -305,6 +320,32 @@ func (c *Client) createDeployment(ctx context.Context, userID string, envVars ma }, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: pointer(NodeGroupID), + RunAsUser: pointer(NodeUserID), + RunAsGroup: pointer(NodeGroupID), + FSGroupChangePolicy: func() *corev1.PodFSGroupChangePolicy { + policy := corev1.FSGroupChangeOnRootMismatch + return &policy + }(), + }, + InitContainers: []corev1.Container{ + { + Name: "volume-permissions", + Image: "busybox:latest", + Command: []string{ + "sh", + "-c", + fmt.Sprintf("mkdir -p /home/nodeuser/.iris && chown -R %d:%d /home/nodeuser/.iris", NodeUserID, NodeGroupID), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "user-data", + MountPath: "/home/nodeuser/.iris", + }, + }, + }, + }, Containers: []corev1.Container{ { Name: "sandbox",