diff --git a/.github/workflows/cri_tests.yml b/.github/workflows/cri_tests.yml index 1fd32d900..6f45a0e6f 100644 --- a/.github/workflows/cri_tests.yml +++ b/.github/workflows/cri_tests.yml @@ -16,7 +16,7 @@ jobs: env: GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_VHIVE_ARGS: "-dbg" - runs-on: ${{ fromJSON(format('["self-hosted", "{0}-cri"]', inputs.sandbox)) }} + runs-on: ubuntu-24.04 steps: - name: Host Info @@ -40,8 +40,35 @@ jobs: sudo apt update sudo apt install rsync -y - - name: Build setup scripts - run: pushd scripts && go build -o setup_tool && popd + - name: Ensure /etc/cni/net.d exists + run: sudo mkdir -p /etc/cni/net.d + + - name: Install containerd and iptables + run: | + sudo apt-get update + sudo apt-get install -y containerd iptables + + - name: Install Docker + run: | + sudo apt-get update + sudo apt-get install -y docker.io + + - name: Start Docker Daemon + run: | + sudo dockerd > /tmp/dockerd.log 2>&1 & + sleep 5 # wait for dockerd to be ready + + - name: Start Minikube + uses: medyagh/setup-minikube@latest + with: + driver: docker + wait: all + start-args: '--force' + + - name: Check cluster status + run: kubectl get nodes + - name: Build setup scripts + run: pushd scripts && go build -o setup_tool && popd - name: Setup vHive CRI test environment run: ./scripts/github_runner/setup_cri_test_env.sh ${{ inputs.sandbox }} @@ -51,7 +78,7 @@ jobs: - name: Run sandbox specific tests if: ${{ inputs.sandbox == 'firecracker' }} - run: source /etc/profile && go clean -testcache && go test ./cri/${{ inputs.sandbox }} -v -race -cover + run: source /etc/profile && go clean -testcache && sudo go test ./cri/${{ inputs.sandbox }} -v -race -cover - name: Archive log artifacts if: ${{ always() }} diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d5815cc54..8def3d387 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,18 +1,19 @@ +--- name: vHive integration tests on: push: - branches: [ main, legacy-firecracker-v0.24.0-with-upf-support ] + branches: [main, legacy-firecracker-v0.24.0-with-upf-support] paths-ignore: - - 'docs/**' - - '**.md' - - 'function-images/**' + - 'docs/**' + - '**.md' + - 'function-images/**' pull_request: - branches: [ main, legacy-firecracker-v0.24.0-with-upf-support ] + branches: [main, legacy-firecracker-v0.24.0-with-upf-support] paths-ignore: - - 'docs/**' - - '**.md' - - 'function-images/**' + - 'docs/**' + - '**.md' + - 'function-images/**' workflow_dispatch: schedule: - cron: '0 19 * * 1' @@ -25,57 +26,133 @@ jobs: integration-tests: name: Integration tests runs-on: ubuntu-24.04 + services: + docker: + image: docker:dind + options: >- + --privileged + ports: + - 2375:2375 + env: + DOCKER_TLS_CERTDIR: '' strategy: fail-fast: false matrix: test-name: [test, test-man-bench] steps: - - uses: actions/checkout@v4 - - name: Handle LFS Caching - uses: ./.github/actions/lfs-cache - with: - lfs_enabled: 'true' - - - name: Set up Go version in go.mod file - uses: actions/setup-go@v5 - with: - go-version-file: ${{ github.workspace }}/go.mod - cache-dependency-path: | - **/go.sum - **/go.mod - - - name: Set up Python 3.x - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Add rsync - run: | - sudo apt update - sudo apt install rsync -y - - - name: Build setup scripts - run: pushd scripts && go build -o setup_tool && popd - - - name: Pull binaries - run: ./scripts/setup_tool setup_firecracker_containerd - - - name: Build - run: go build -race -v -a ./... - - - name: Run vHive end-to-end tests - run: make ${{ matrix.test-name }} - - - name: Archive log artifacts - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: ctrd-logs-${{ matrix.test-name }} - path: | - /tmp/ctrd-logs/ - ${{ github.workspace }}/*.log - ${{ github.workspace }}/scripts/github_runner/*.log - - - name: Cleaning - if: ${{ always() }} - run: ./scripts/setup_tool clean_fcctr + - uses: actions/checkout@v4 + + - name: Handle LFS Caching + uses: ./.github/actions/lfs-cache + with: + lfs_enabled: true + + - name: Set up Go version in go.mod file + uses: actions/setup-go@v5 + with: + go-version-file: ${{ github.workspace }}/go.mod + cache-dependency-path: | + **/go.sum + **/go.mod + + - name: Set up Python 3.x + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Add rsync and other dependencies + run: | + sudo apt-get update + sudo apt-get install -y rsync fuse-overlayfs cni-plugins + + #- name: Ensure /etc/cni/net.d exists + # run: sudo mkdir -p /etc/cni/net.d + + #- name: Install containerd and iptables + # run: | + # sudo apt-get update + # sudo apt-get install -y containerd iptables + + # Use system Docker, do not install or run your own daemon! + # GitHub runners provide Docker pre-installed, fully privileged. + # Uncomment next two steps ONLY IF YOU REALLY NEED DIN, otherwise use system docker. + + # - name: Install Docker + # run: | + # sudo apt-get update + # sudo apt-get install -y docker.io + + # - name: Start Docker Daemon (DinD, not recommended on hosted runners) + # run: | + # sudo dockerd --storage-driver=overlay2 > /tmp/dockerd.log 2>&1 & + # sleep 5 + # tries=0 + # until docker info >/dev/null 2>&1; do + # if (( tries > 15 )); then + # echo "Docker daemon failed to start"; tail -50 /tmp/dockerd.log; exit 1 + # fi + # sleep 2; ((tries++)) + + - name: Start Docker daemon + run: | + dockerd-entrypoint.sh & + sleep 10 + # done + + - name: Show Docker version/info + run: | + docker version + docker info + + - name: Clean up Docker networks + run: | + docker network prune -f + docker system prune -f + + - name: Start Minikube + uses: medyagh/setup-minikube@latest + with: + driver: docker + wait: all + start-args: '--force' + + - name: Check cluster status + run: kubectl get nodes + + - name: Collect Docker and iptables logs for debug + if: failure() + run: | + sudo journalctl -u docker || true + docker info || true + ps auxf | grep dockerd || true + sudo iptables -t nat -L -v -n || true + tail -50 /tmp/dockerd.log || true + + - name: Ensure kubeconfig set + run: echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV + + - name: Build setup scripts + run: pushd scripts && go build -o setup_tool && popd + + - name: Pull binaries + run: ./scripts/setup_tool setup_firecracker_containerd + + - name: Build + run: go build -race -v -a ./... + + - name: Run vHive end-to-end tests + run: make ${{ matrix.test-name }} + + - name: Archive log artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: ctrd-logs-${{ matrix.test-name }} + path: | + /tmp/ctrd-logs/ + ${{ github.workspace }}/*.log + ${{ github.workspace }}/scripts/github_runner/*.log + + - name: Cleaning + if: ${{ always() }} + run: ./scripts/setup_tool clean_fcctr diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 579ebf0c0..cac5d74b6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -69,7 +69,7 @@ jobs: profile-unit-test: name: "Unit test: profile unit test" - runs-on: [self-hosted, profile] + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -147,6 +147,34 @@ jobs: - name: Cleanup runner disk space run: ./scripts/github_runner/gh-actions-disk-cleanup.sh + - name: Ensure /etc/cni/net.d exists + run: sudo mkdir -p /etc/cni/net.d + + - name: Install containerd and iptables + run: | + sudo apt-get update + sudo apt-get install -y containerd iptables + + - name: Install Docker + run: | + sudo apt-get update + sudo apt-get install -y docker.io + + - name: Start Docker Daemon + run: | + sudo dockerd > /tmp/dockerd.log 2>&1 & + sleep 5 # wait for dockerd to be ready + + - name: Start Minikube + uses: medyagh/setup-minikube@latest + with: + driver: docker + wait: all + start-args: '--force' + + - name: Check cluster status + run: kubectl get nodes + - name: Build setup scripts run: pushd scripts && go build -o setup_tool && popd diff --git a/configs/knative_workloads/gvisor/helloworld.yaml b/configs/knative_workloads/gvisor/helloworld.yaml index 80f60754e..fbb903a7a 100644 --- a/configs/knative_workloads/gvisor/helloworld.yaml +++ b/configs/knative_workloads/gvisor/helloworld.yaml @@ -6,7 +6,7 @@ spec: template: spec: containers: - - image: ghcr.io/ease-lab/helloworld:var_workload + - image: ghcr.io/ease-lab/helloworld@sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25 ports: - name: h2c # For GRPC support containerPort: 50051 @@ -14,4 +14,4 @@ spec: - name: GUEST_PORT # Port on which the firecracker-containerd container is accepting requests value: "50051" - name: GUEST_IMAGE # Container image to use for firecracker-containerd container - value: "ghcr.io/ease-lab/helloworld:var_workload" + value: "ghcr.io/ease-lab/helloworld@sha256:12dc6715ed1a8306f246ceaf7742c09e38a52a79d17421e4a50d7e0e09fdbc25" diff --git a/scripts/github_runner/setup_cri_test_env.sh b/scripts/github_runner/setup_cri_test_env.sh index fe83ccfd4..eeba4550f 100755 --- a/scripts/github_runner/setup_cri_test_env.sh +++ b/scripts/github_runner/setup_cri_test_env.sh @@ -35,16 +35,21 @@ fi SANDBOX=$1 VHIVE_ROOT="$(git rev-parse --show-toplevel)" +echo "=== Setting-Up onenode-vhive ===" $VHIVE_ROOT/scripts/setup_tool -vhive-repo-dir $VHIVE_ROOT start_onenode_vhive_cluster $SANDBOX # $VHIVE_ROOT/scripts/cloudlab/start_onenode_vhive_cluster.sh "$SANDBOX" sleep 30s +echo "=== Setting-Up zipkin ===" # KUBECONFIG=/etc/kubernetes/admin.conf sudo $VHIVE_ROOT/scripts/setup_zipkin.sh $VHIVE_ROOT/scripts/setup_tool -vhive-repo-dir $VHIVE_ROOT setup_zipkin # FIXME (gh-709) #source etc/profile && go run $VHIVE_ROOT/examples/registry/populate_registry.go -imageFile $VHIVE_ROOT/examples/registry/images.txt +echo "=== Activator Status ===" && sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl describe pod -n knative-serving -l app=activator +echo "=== Activator Logs ===" && sudo KUBECONFIG=/etc/kubernetes/admin.conf kubectl logs -n knative-serving -l app=activator + sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply helloworld -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/helloworld.yaml sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply helloworldserial -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/helloworldSerial.yaml sudo KUBECONFIG=/etc/kubernetes/admin.conf kn service apply pyaes -f $VHIVE_ROOT/configs/knative_workloads/$SANDBOX/pyaes.yaml diff --git a/snapshotting/manager.go b/snapshotting/manager.go index ca796da86..fffd9b8d4 100644 --- a/snapshotting/manager.go +++ b/snapshotting/manager.go @@ -41,15 +41,19 @@ type SnapshotManager struct { } // Snapshot identified by VM id - func NewSnapshotManager(baseFolder string) *SnapshotManager { manager := new(SnapshotManager) manager.snapshots = make(map[string]*Snapshot) manager.baseFolder = baseFolder - // Clean & init basefolder - _ = os.RemoveAll(manager.baseFolder) - _ = os.MkdirAll(manager.baseFolder, os.ModePerm) + if err := os.RemoveAll(manager.baseFolder); err != nil { + log.Printf("Failed to clean base folder: %v", err) + return nil + } + if err := os.MkdirAll(manager.baseFolder, 0755); err != nil { + log.Printf("Failed to create base folder: %v", err) + return nil + } return manager }