feat: add jailer startup test with parallelism

Add another test of jailer startup time, now
with variable number of parallel launches of the jailer
process.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>

Author: ShadowCurse

tests/integration_tests/performance/test_jailer.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 import subprocess
+from concurrent.futures import ProcessPoolExecutor
 
 import pytest
 
@@ -83,3 +84,83 @@ def test_jailer_startup(
     for d in os.listdir(mounts_paths):
         subprocess.run(["umount", f"{mounts_paths}/{d}"], check=True)
     shutil.rmtree(DEFAULT_CHROOT_PATH)
+
+
+def run_command(cmd):
+    """
+    Run the cmd in subprocess. Needs to be outside
+    other functions, otherwise the ProcessPoolExecutor
+    will not be able to pickle it.
+    """
+    return subprocess.run(
+        cmd,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+
+@pytest.mark.nonci
+@pytest.mark.parametrize("parallel", [1, 5, 10])
+@pytest.mark.parametrize("mounts", [0, 100, 300, 500])
+def test_jailer_startup_parallel(
+    jailer_time_bin, tmp_path, microvm_factory, parallel, mounts, metrics
+):
+    """
+    Test the overhead of jailer startup without and with bind mounts
+    with different parallelism options.
+    """
+
+    jailer_binary = microvm_factory.jailer_binary_path
+
+    # Create bind mount points. The exact location of them
+    # does not matter, they just need to exist.
+    mounts_paths = tmp_path / "mounts"
+    os.makedirs(mounts_paths)
+    for m in range(mounts):
+        mount_path = f"{mounts_paths}/mount{m}"
+        os.makedirs(mount_path)
+        subprocess.run(
+            ["mount", "--bind", f"{mount_path}", f"{mount_path}"], check=True
+        )
+
+    metrics.set_dimensions(
+        {
+            "instance": global_props.instance,
+            "cpu_model": global_props.cpu_model,
+            "performance_test": "test_jailer_startup_parallel",
+            "parallel": str(parallel),
+            "mounts": str(mounts),
+        }
+    )
+
+    cmds = []
+    for i in range(500):
+        jailer = JailerContext(
+            jailer_id=f"fakefc{i}",
+            exec_file=jailer_time_bin,
+            # Don't deamonize to get the stdout
+            daemonize=False,
+        )
+        jailer.setup()
+
+        cmd = [str(jailer_binary), *jailer.construct_param_list()]
+        cmds.append(cmd)
+
+    with ProcessPoolExecutor(max_workers=parallel) as executor:
+        # Submit all commands and get results
+        results = list(executor.map(run_command, cmds))
+
+        # Get results as they complete
+        for result in results:
+            e, s = result.stdout.split()
+            metrics.put_metric(
+                "startup",
+                int(e) - int(s),
+                unit="Microseconds",
+            )
+
+    # Cleanup mounts and jailer dirs
+    for d in os.listdir(mounts_paths):
+        subprocess.run(["umount", f"{mounts_paths}/{d}"], check=True)
+    shutil.rmtree(DEFAULT_CHROOT_PATH)