Add Darwin support via vfkit hypervisor

flake.nix

@@ -156,7 +156,8 @@
               # currently broken:
               # "crosvm"
             ];
-            hypervisorsWithUserNet = [ "qemu" "kvmtool" ];
+            hypervisorsWithUserNet = [ "qemu" "kvmtool" "vfkit" ];
+            hypervisorsDarwinOnly = [ "vfkit" ];
             makeExample = { system, hypervisor, config ? {} }:
               nixpkgs.lib.nixosSystem {
                 system = nixpkgs.lib.replaceString "-darwin" "-linux" system;
@@ -172,12 +173,21 @@
                     nixpkgs.overlays = [ self.overlay ];
                     microvm = {
                       inherit hypervisor;
-                      # share the host's /nix/store if the hypervisor can do 9p
-                      shares = lib.optional (builtins.elem hypervisor hypervisorsWith9p) {
-                        tag = "ro-store";
-                        source = "/nix/store";
-                        mountPoint = "/nix/.ro-store";
-                      };
+                      # share the host's /nix/store if the hypervisor supports it
+                      shares =
+                        if builtins.elem hypervisor hypervisorsWith9p then [{
+                          tag = "ro-store";
+                          source = "/nix/store";
+                          mountPoint = "/nix/.ro-store";
+                          proto = "9p";
+                        }]
+                        else if hypervisor == "vfkit" then [{
+                          tag = "ro-store";
+                          source = "/nix/store";
+                          mountPoint = "/nix/.ro-store";
+                          proto = "virtiofs";
+                        }]
+                        else [];
                       # writableStoreOverlay = "/nix/.rw-store";
                       # volumes = [ {
                       #   image = "nix-store-overlay.img";
@@ -208,34 +218,44 @@
               };
           in
             (builtins.foldl' (results: system:
-              builtins.foldl' ({ result, n }: hypervisor: {
-                result = result // {
-                  "${system}-${hypervisor}-example" = makeExample {
-                    inherit system hypervisor;
-                  };
-                } //
-                nixpkgs.lib.optionalAttrs (builtins.elem hypervisor self.lib.hypervisorsWithNetwork) {
-                  "${system}-${hypervisor}-example-with-tap" = makeExample {
-                    inherit system hypervisor;
-                    config = _: {
-                      microvm.interfaces = [ {
-                        type = "tap";
-                        id = "vm-${builtins.substring 0 4 hypervisor}";
-                        mac = "02:00:00:01:01:0${toString n}";
-                      } ];
-                      networking = {
-                        interfaces.eth0.useDHCP = true;
-                        firewall.allowedTCPPorts = [ 22 ];
-                      };
-                      services.openssh = {
-                        enable = true;
-                        settings.PermitRootLogin = "yes";
+              builtins.foldl' ({ result, n }: hypervisor:
+                let
+                  # Skip darwin-only hypervisors on Linux systems
+                  isDarwinOnly = builtins.elem hypervisor hypervisorsDarwinOnly;
+                  isDarwinSystem = nixpkgs.lib.hasSuffix "-darwin" system;
+                  shouldSkip = isDarwinOnly && !isDarwinSystem;
+                in
+                if shouldSkip then { inherit result n; }
+                else {
+                  result = result // {
+                    "${system}-${hypervisor}-example" = makeExample {
+                      inherit system hypervisor;
+                    };
+                  } //
+                  # Skip tap example for darwin-only hypervisors (vfkit doesn't support tap)
+                  nixpkgs.lib.optionalAttrs (builtins.elem hypervisor self.lib.hypervisorsWithNetwork && !isDarwinOnly) {
+                    "${system}-${hypervisor}-example-with-tap" = makeExample {
+                      inherit system hypervisor;
+                      config = _: {
+                        microvm.interfaces = [ {
+                          type = "tap";
+                          id = "vm-${builtins.substring 0 4 hypervisor}";
+                          mac = "02:00:00:01:01:0${toString n}";
+                        } ];
+                        networking = {
+                          interfaces.eth0.useDHCP = true;
+                          firewall.allowedTCPPorts = [ 22 ];
+                        };
+                        services.openssh = {
+                          enable = true;
+                          settings.PermitRootLogin = "yes";
+                        };
                       };
                     };
                   };
-                };
-                n = n + 1;
-              }) results self.lib.hypervisors
+                  n = n + 1;
+                }
+              ) results self.lib.hypervisors
             ) { result = {}; n = 1; } systems).result;
       };
 }

lib/default.nix

@@ -8,6 +8,7 @@ rec {
     "kvmtool"
     "stratovirt"
     "alioth"
+    "vfkit"
   ];
 
   hypervisorsWithNetwork = hypervisors;

lib/runners/vfkit.nix

@@ -0,0 +1,132 @@
+{ pkgs
+, microvmConfig
+, macvtapFds
+, withDriveLetters
+, ...
+}:
+
+let
+  inherit (pkgs) lib;
+  inherit (vmHostPackages.stdenv.hostPlatform) system;
+  inherit (microvmConfig) vmHostPackages;
+
+  vfkit = vmHostPackages.vfkit;
+
+  inherit (microvmConfig)
+    hostName vcpu mem user interfaces volumes shares socket
+    storeOnDisk kernel initrdPath storeDisk kernelParams
+    balloon devices credentialFiles vsock;
+
+  inherit (microvmConfig.vfkit) extraArgs logLevel;
+
+  volumesWithLetters = withDriveLetters microvmConfig;
+
+  # vfkit requires uncompressed kernel
+  kernelPath = "${kernel.out}/${pkgs.stdenv.hostPlatform.linux-kernel.target}";
+
+  kernelCmdLine = "console=hvc0 reboot=t panic=-1 ${toString kernelParams}";
+
+  bootloaderArgs = [
+    "--bootloader"
+    "linux,kernel=${kernelPath},initrd=${initrdPath},cmdline=\"${kernelCmdLine}\""
+  ];
+
+  deviceArgs =
+    [ "--device" "virtio-rng" ]
+    ++
+    [ "--device" "virtio-serial,stdio" ]
+    ++
+    (builtins.concatMap ({ image, ... }: [
+      "--device" "virtio-blk,path=${image}"
+    ]) volumesWithLetters)
+    ++
+    (builtins.concatMap ({ proto, source, tag, ... }:
+      if proto == "virtiofs" then [
+        "--device" "virtio-fs,sharedDir=${source},mountTag=${tag}"
+      ]
+      else if proto == "9p" then
+        throw "vfkit does not support 9p shares on macOS. Use proto = \"virtiofs\" instead."
+      else
+        throw "Unknown share protocol: ${proto}"
+    ) shares)
+    ++
+    (builtins.concatMap ({ type, id, mac, ... }:
+      if type == "user" then [
+        "--device" "virtio-net,nat,mac=${mac}"
+      ]
+      else if type == "tap" then
+        throw "vfkit does not support tap networking on macOS. Use type = \"user\" for NAT networking."
+      else if type == "bridge" then
+        throw "vfkit bridge networking requires vmnet-helper which is not yet implemented. Use type = \"user\" for NAT networking."
+      else if type == "macvtap" then
+        throw "vfkit does not support macvtap networking on macOS. Use type = \"user\" for NAT networking."
+      else
+        throw "Unknown network interface type: ${type}"
+    ) interfaces);
+
+  allArgsWithoutSocket = [
+    "${vfkit}/bin/vfkit"
+    "--cpus" (toString vcpu)
+    "--memory" (toString mem)
+  ]
+  ++ lib.optionals (logLevel != null) [
+    "--log-level" logLevel
+  ]
+  ++ bootloaderArgs
+  ++ deviceArgs
+  ++ extraArgs;
+
+in
+{
+  tapMultiQueue = false;
+
+  preStart = lib.optionalString (socket != null) ''
+    rm -f ${socket}
+  '';
+
+  command =
+    if !vmHostPackages.stdenv.hostPlatform.isDarwin
+    then throw "vfkit only works on macOS (Darwin). Current host: ${system}"
+    else if vmHostPackages.stdenv.hostPlatform.isAarch64 != pkgs.stdenv.hostPlatform.isAarch64
+    then throw "vfkit requires matching host and guest architectures. Host: ${system}, Guest: ${pkgs.stdenv.hostPlatform.system}"
+    else if user != null
+    then throw "vfkit does not support changing user"
+    else if balloon
+    then throw "vfkit does not support memory ballooning"
+    else if devices != []
+    then throw "vfkit does not support device passthrough"
+    else if credentialFiles != {}
+    then throw "vfkit does not support credentialFiles"
+    else if vsock.cid != null
+    then throw "vfkit vsock support not yet implemented in microvm.nix"
+    else if storeOnDisk
+    then throw "vfkit does not support storeOnDisk. Use virtiofs shares instead (already configured in examples)."
+    else
+      let
+        baseCmd = lib.escapeShellArgs allArgsWithoutSocket;
+        vfkitCmd = lib.concatStringsSep " " (map lib.escapeShellArg allArgsWithoutSocket);
+      in
+      # vfkit requires absolute socket paths, so expand relative paths
+      if socket != null
+      then "bash -c ${lib.escapeShellArg ''
+        SOCKET_ABS=${lib.escapeShellArg socket}
+        [[ "$SOCKET_ABS" != /* ]] && SOCKET_ABS="$PWD/$SOCKET_ABS"
+        exec ${vfkitCmd} --restful-uri "unix:///$SOCKET_ABS"
+      ''}"
+      else baseCmd;
+
+  canShutdown = socket != null;
+
+  shutdownCommand =
+    if socket != null
+    then ''
+      SOCKET_ABS="${lib.escapeShellArg socket}"
+      [[ "$SOCKET_ABS" != /* ]] && SOCKET_ABS="$PWD/$SOCKET_ABS"
+      echo '{"state": "Stop"}' | ${vmHostPackages.socat}/bin/socat - "UNIX-CONNECT:$SOCKET_ABS"
+    ''
+    else throw "Cannot shutdown without socket";
+
+  supportsNotifySocket = false;
+
+  requiresMacvtapAsFds = false;
+}

nixos-modules/microvm/options.nix

@@ -590,6 +590,18 @@ in
       description = "Custom CPU template passed to firecracker.";
     };
 
+    vfkit.extraArgs = mkOption {
+      type = with types; listOf str;
+      default = [];
+      description = "Extra arguments to pass to vfkit.";
+    };
+
+    vfkit.logLevel = mkOption {
+      type = with types; nullOr (enum ["debug" "info" "error"]);
+      default = "info";
+      description = "vfkit log level.";
+    };
+
     prettyProcnames = mkOption {
       type = types.bool;
       default = true;