Migrates remaining tests to github runners

.github/workflows/cri_tests.yml

@@ -16,7 +16,7 @@ jobs:
     env:
       GITHUB_RUN_ID: ${{ github.run_id }}
       GITHUB_VHIVE_ARGS: "-dbg"
-    runs-on: ${{ fromJSON(format('["self-hosted", "{0}-cri"]', inputs.sandbox)) }}
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Host Info
@@ -40,8 +40,35 @@ jobs:
           sudo apt update
           sudo apt install rsync -y
 
-      - name: Build setup scripts
-        run: pushd scripts && go build -o setup_tool && popd
+      - name: Ensure /etc/cni/net.d exists
+        run: sudo mkdir -p /etc/cni/net.d
+
+      - name: Install containerd and iptables
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y containerd iptables
+
+      - name: Install Docker
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker.io
+
+      - name: Start Docker Daemon
+        run: |
+          sudo dockerd > /tmp/dockerd.log 2>&1 &
+          sleep 5  # wait for dockerd to be ready
+
+      - name: Start Minikube
+        uses: medyagh/setup-minikube@latest
+        with:
+          driver: docker
+          wait: all
+          start-args: '--force'
+
+      - name: Check cluster status
+        run: kubectl get nodes
+        - name: Build setup scripts
+          run: pushd scripts && go build -o setup_tool && popd
 
       - name: Setup vHive CRI test environment
         run: ./scripts/github_runner/setup_cri_test_env.sh ${{ inputs.sandbox }}
@@ -51,7 +78,7 @@ jobs:
 
       - name: Run sandbox specific tests
         if: ${{ inputs.sandbox == 'firecracker' }}
-        run: source /etc/profile && go clean -testcache && go test ./cri/${{ inputs.sandbox }} -v -race -cover
+        run: source /etc/profile && go clean -testcache && sudo go test ./cri/${{ inputs.sandbox }} -v -race -cover
 
       - name: Archive log artifacts
         if: ${{ always() }}

.github/workflows/integration_tests.yml

@@ -1,18 +1,19 @@
+---
 name: vHive integration tests
 
 on:
   push:
-    branches: [ main, legacy-firecracker-v0.24.0-with-upf-support ]
+    branches: [main, legacy-firecracker-v0.24.0-with-upf-support]
     paths-ignore:
-    - 'docs/**'
-    - '**.md'
-    - 'function-images/**'
+      - 'docs/**'
+      - '**.md'
+      - 'function-images/**'
   pull_request:
-    branches: [ main, legacy-firecracker-v0.24.0-with-upf-support ]
+    branches: [main, legacy-firecracker-v0.24.0-with-upf-support]
     paths-ignore:
-    - 'docs/**'
-    - '**.md'
-    - 'function-images/**'
+      - 'docs/**'
+      - '**.md'
+      - 'function-images/**'
   workflow_dispatch:
   schedule:
     - cron: '0 19 * * 1'
@@ -25,57 +26,133 @@ jobs:
   integration-tests:
     name: Integration tests
     runs-on: ubuntu-24.04
+    services:
+      docker:
+        image: docker:dind
+        options: >-
+          --privileged
+        ports:
+          - 2375:2375
+        env:
+          DOCKER_TLS_CERTDIR: ''
     strategy:
       fail-fast: false
       matrix:
         test-name: [test, test-man-bench]
     steps:
-    - uses: actions/checkout@v4
-    - name: Handle LFS Caching
-      uses: ./.github/actions/lfs-cache
-      with:
-        lfs_enabled: 'true'
-
-    - name: Set up Go version in go.mod file
-      uses: actions/setup-go@v5
-      with:
-        go-version-file: ${{ github.workspace }}/go.mod
-        cache-dependency-path: |
-          **/go.sum
-          **/go.mod
-
-    - name: Set up Python 3.x
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-
-    - name: Add rsync
-      run: |
-        sudo apt update
-        sudo apt install rsync -y
-
-    - name: Build setup scripts
-      run: pushd scripts && go build -o setup_tool && popd
-
-    - name: Pull binaries
-      run: ./scripts/setup_tool setup_firecracker_containerd
-
-    - name: Build
-      run: go build -race -v -a ./...
-
-    - name: Run vHive end-to-end tests
-      run: make ${{ matrix.test-name }}
-
-    - name: Archive log artifacts
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v4
-      with:
-        name: ctrd-logs-${{ matrix.test-name }}
-        path: |
-          /tmp/ctrd-logs/
-          ${{ github.workspace }}/*.log
-          ${{ github.workspace }}/scripts/github_runner/*.log
-
-    - name: Cleaning
-      if: ${{ always() }}
-      run: ./scripts/setup_tool clean_fcctr
+      - uses: actions/checkout@v4
+
+      - name: Handle LFS Caching
+        uses: ./.github/actions/lfs-cache
+        with:
+          lfs_enabled: true
+
+      - name: Set up Go version in go.mod file
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: ${{ github.workspace }}/go.mod
+          cache-dependency-path: |
+            **/go.sum
+            **/go.mod
+
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Add rsync and other dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y rsync fuse-overlayfs cni-plugins
+
+      #- name: Ensure /etc/cni/net.d exists
+      #  run: sudo mkdir -p /etc/cni/net.d
+
+      #- name: Install containerd and iptables
+      #  run: |
+      #    sudo apt-get update
+      #    sudo apt-get install -y containerd iptables
+
+      # Use system Docker, do not install or run your own daemon!
+      # GitHub runners provide Docker pre-installed, fully privileged.
+      # Uncomment next two steps ONLY IF YOU REALLY NEED DIN, otherwise use system docker.
+
+      # - name: Install Docker
+      #   run: |
+      #     sudo apt-get update
+      #     sudo apt-get install -y docker.io
+
+      # - name: Start Docker Daemon (DinD, not recommended on hosted runners)
+      #   run: |
+      #     sudo dockerd --storage-driver=overlay2 > /tmp/dockerd.log 2>&1 &
+      #     sleep 5
+      #     tries=0
+      #     until docker info >/dev/null 2>&1; do
+      #       if (( tries > 15 )); then
+      #         echo "Docker daemon failed to start"; tail -50 /tmp/dockerd.log; exit 1
+      #       fi
+      #       sleep 2; ((tries++))
+
+      - name: Start Docker daemon
+        run: |
+          dockerd-entrypoint.sh &
+          sleep 10
+      #     done
+
+      - name: Show Docker version/info
+        run: |
+          docker version
+          docker info
+
+      - name: Clean up Docker networks
+        run: |
+          docker network prune -f
+          docker system prune -f
+
+      - name: Start Minikube
+        uses: medyagh/setup-minikube@latest
+        with:
+          driver: docker
+          wait: all
+          start-args: '--force'
+
+      - name: Check cluster status
+        run: kubectl get nodes
+
+      - name: Collect Docker and iptables logs for debug
+        if: failure()
+        run: |
+          sudo journalctl -u docker || true
+          docker info || true
+          ps auxf | grep dockerd || true
+          sudo iptables -t nat -L -v -n || true
+          tail -50 /tmp/dockerd.log || true
+
+      - name: Ensure kubeconfig set
+        run: echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV
+
+      - name: Build setup scripts
+        run: pushd scripts && go build -o setup_tool && popd
+
+      - name: Pull binaries
+        run: ./scripts/setup_tool setup_firecracker_containerd
+
+      - name: Build
+        run: go build -race -v -a ./...
+
+      - name: Run vHive end-to-end tests
+        run: make ${{ matrix.test-name }}
+
+      - name: Archive log artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ctrd-logs-${{ matrix.test-name }}
+          path: |
+            /tmp/ctrd-logs/
+            ${{ github.workspace }}/*.log
+            ${{ github.workspace }}/scripts/github_runner/*.log
+
+      - name: Cleaning
+        if: ${{ always() }}
+        run: ./scripts/setup_tool clean_fcctr

.github/workflows/unit_tests.yml

@@ -69,7 +69,7 @@ jobs:
 
   profile-unit-test:
     name: "Unit test: profile unit test"
-    runs-on: [self-hosted, profile]
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -147,6 +147,34 @@ jobs:
     - name: Cleanup runner disk space
       run: ./scripts/github_runner/gh-actions-disk-cleanup.sh
 
+    - name: Ensure /etc/cni/net.d exists
+      run: sudo mkdir -p /etc/cni/net.d
+
+    - name: Install containerd and iptables
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y containerd iptables
+
+    - name: Install Docker
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y docker.io
+
+    - name: Start Docker Daemon
+      run: |
+        sudo dockerd > /tmp/dockerd.log 2>&1 &
+        sleep 5  # wait for dockerd to be ready
+
+    - name: Start Minikube
+      uses: medyagh/setup-minikube@latest
+      with:
+        driver: docker
+        wait: all
+        start-args: '--force'
+
+    - name: Check cluster status
+      run: kubectl get nodes
+
     - name: Build setup scripts
       run: pushd scripts && go build -o setup_tool && popd
 

configs/knative_workloads/gvisor/helloworld.yaml

@@ -6,12 +6,12 @@ spec:
   template:
     spec:
       containers:
-        - image: ghcr.io/ease-lab/helloworld:var_workload
+        - image: ghcr.io/ease-lab/helloworld@sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25
           ports:
             - name: h2c # For GRPC support
               containerPort: 50051
           env:
             - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests
               value: "50051"
             - name: GUEST_IMAGE # Container image to use for firecracker-containerd container
-              value: "ghcr.io/ease-lab/helloworld:var_workload"
+              value: "ghcr.io/ease-lab/helloworld@sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25"

scripts/github_runner/setup_cri_test_env.sh

@@ -35,16 +35,21 @@ fi
 SANDBOX=$1
 VHIVE_ROOT="$(git rev-parse --show-toplevel)"
 
+echo "=== Setting-Up onenode-vhive ==="
 $VHIVE_ROOT/scripts/setup_tool -vhive-repo-dir $VHIVE_ROOT start_onenode_vhive_cluster $SANDBOX
 # $VHIVE_ROOT/scripts/cloudlab/start_onenode_vhive_cluster.sh "$SANDBOX"
 sleep 30s
 
+echo "=== Setting-Up zipkin ==="
 # KUBECONFIG=/etc/kubernetes/admin.conf sudo $VHIVE_ROOT/scripts/setup_zipkin.sh
 $VHIVE_ROOT/scripts/setup_tool -vhive-repo-dir $VHIVE_ROOT setup_zipkin
 
 # FIXME (gh-709)
 #source etc/profile && go run $VHIVE_ROOT/examples/registry/populate_registry.go -imageFile $VHIVE_ROOT/examples/registry/images.txt
 
+echo "=== Activator Status ===" && sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl describe pod -n knative-serving -l app=activator
+echo "=== Activator Logs ===" && sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl logs -n knative-serving -l app=activator
+
 sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply helloworld -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/helloworld.yaml
 sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply helloworldserial -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/helloworldSerial.yaml
 sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply pyaes -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/pyaes.yaml

snapshotting/manager.go

@@ -41,15 +41,19 @@ type SnapshotManager struct {
 }
 
 // Snapshot identified by VM id
-
 func NewSnapshotManager(baseFolder string) *SnapshotManager {
 	manager := new(SnapshotManager)
 	manager.snapshots = make(map[string]*Snapshot)
 	manager.baseFolder = baseFolder
 
-	// Clean & init basefolder
-	_ = os.RemoveAll(manager.baseFolder)
-	_ = os.MkdirAll(manager.baseFolder, os.ModePerm)
+	if err := os.RemoveAll(manager.baseFolder); err != nil {
+		log.Printf("Failed to clean base folder: %v", err)
+		return nil
+	}
+	if err := os.MkdirAll(manager.baseFolder, 0755); err != nil {
+		log.Printf("Failed to create base folder: %v", err)
+		return nil
+	}
 
 	return manager
 }