Merge remote-tracking branch 'upstream/main' into rustify-snapshot-module

Author: beamandala

.buildkite/common.py

@@ -13,14 +13,14 @@
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = [
-    "c5n.metal",  # Intel Skylake
-    "m5n.metal",  # Intel Cascade Lake
-    "m6i.metal",  # Intel Icelake
-    "m6a.metal",  # AMD Milan
-    "m6g.metal",  # Graviton2
-    "m7g.metal",  # Graviton3
-]
+DEFAULT_INSTANCES = {
+    "c5n.metal": "x86_64",  # Intel Skylake
+    "m5n.metal": "x86_64",  # Intel Cascade Lake
+    "m6i.metal": "x86_64",  # Intel Icelake
+    "m6a.metal": "x86_64",  # AMD Milan
+    "m6g.metal": "aarch64",  # Graviton2
+    "m7g.metal": "aarch64",  # Graviton3
+}
 
 DEFAULT_PLATFORMS = [
     ("al2", "linux_5.10"),
@@ -145,7 +145,7 @@ def __call__(self, parser, namespace, value, option_string=None):
     "--instances",
     required=False,
     nargs="+",
-    default=DEFAULT_INSTANCES,
+    default=DEFAULT_INSTANCES.keys(),
 )
 COMMON_PARSER.add_argument(
     "--platforms",
@@ -188,6 +188,7 @@ def ab_revision_build(revision):
         "git branch $$branch_name $$commitish",
         f"git clone -b $$branch_name . build/{revision}",
         f"cd build/{revision} && ./tools/devtool -y build --release && cd -",
+        "git branch -D $$branch_name",
     ]
 
 
@@ -204,7 +205,9 @@ def shared_build():
     if rev_a is not None:
         rev_b = os.environ.get("REVISION_B")
         assert rev_b is not None, "REVISION_B environment variable not set"
-        build_cmds = ab_revision_build(rev_a) + ab_revision_build(rev_b)
+        build_cmds = ab_revision_build(rev_a)
+        if rev_a != rev_b:
+            build_cmds += ab_revision_build(rev_b)
     elif os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false":
         build_cmds = ab_revision_build(
             os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
@@ -229,7 +232,7 @@ class BKPipeline:
 
     parser = COMMON_PARSER
 
-    def __init__(self, initial_steps=None, **kwargs):
+    def __init__(self, with_build_step=True, **kwargs):
         self.steps = []
         self.args = args = self.parser.parse_args()
         # Retry one time if agent was lost. This can happen if we terminate the
@@ -252,42 +255,36 @@ def __init__(self, initial_steps=None, **kwargs):
         self.per_arch["platforms"] = [("al2", "linux_5.10")]
         self.binary_dir = args.binary_dir
         # Build sharing
-        build_cmds, self.shared_build = shared_build()
-        step_build = group("🏗️ Build", build_cmds, **self.per_arch)
-        self.steps += [step_build, "wait"]
-
-        # If we run initial_steps before the "wait" step above, then a failure of the initial steps
-        # would result in the build not progressing past the "wait" step (as buildkite only proceeds past a wait step
-        # if everything before it passed). Thus put the initial steps after the "wait" step, but set `"depends_on": null`
-        # to start running them immediately (e.g. without waiting for the "wait" step to unblock).
-        #
-        # See also https://buildkite.com/docs/pipelines/dependencies#explicit-dependencies-in-uploaded-steps
-        if initial_steps:
-            for step in initial_steps:
-                step["depends_on"] = None
-
-            self.steps += initial_steps
-
-    def add_step(self, step, decorate=True):
+        if with_build_step:
+            build_cmds, self.shared_build = shared_build()
+            self.build_group_per_arch(
+                "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+            )
+        else:
+            self.shared_build = None
+
+    def add_step(self, step, depends_on_build=True):
         """
         Add a step to the pipeline.
 
         https://buildkite.com/docs/pipelines/step-reference
 
         :param step: a Buildkite step
-        :param decorate: inject needed commands for sharing builds
+        :param depends_on_build: inject needed commands for sharing builds
         """
-        if decorate and isinstance(step, dict):
+        if depends_on_build and isinstance(step, dict):
             step = self._adapt_group(step)
         self.steps.append(step)
         return step
 
     def _adapt_group(self, group):
         """"""
-        prepend = [
-            f'buildkite-agent artifact download "{self.shared_build}" .',
-            f"tar xzf {self.shared_build}",
-        ]
+        prepend = []
+        if self.shared_build is not None:
+            prepend = [
+                f'buildkite-agent artifact download "{self.shared_build}" .',
+                f"tar xzf {self.shared_build}",
+            ]
         if self.binary_dir is not None:
             prepend.extend(
                 [
@@ -298,6 +295,10 @@ def _adapt_group(self, group):
 
         for step in group["steps"]:
             step["command"] = prepend + step["command"]
+            if self.shared_build is not None:
+                step["depends_on"] = self.build_key(
+                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                )
         return group
 
     def build_group(self, *args, **kwargs):
@@ -306,15 +307,34 @@ def build_group(self, *args, **kwargs):
 
         https://buildkite.com/docs/pipelines/group-step
         """
+        depends_on_build = kwargs.pop("depends_on_build", True)
         combined = overlay_dict(self.per_instance, kwargs)
-        return self.add_step(group(*args, **combined))
+        return self.add_step(
+            group(*args, **combined), depends_on_build=depends_on_build
+        )
+
+    def build_key(self, arch):
+        """Return the Buildkite key for the build step, for the specified arch"""
+        return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
-    def build_group_per_arch(self, *args, **kwargs):
+    def build_group_per_arch(self, label, *args, **kwargs):
         """
         Build a group, parametrizing over the architectures only.
+
+        kwargs consumed by this method and not passed down to `group`:
+        - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
+        - `set_key`: If True, causes the generated steps to have a "key" field
         """
+        depends_on_build = kwargs.pop("depends_on_build", True)
+        set_key = kwargs.pop("set_key", None)
         combined = overlay_dict(self.per_arch, kwargs)
-        return self.add_step(group(*args, **combined))
+        grp = group(label, *args, **combined)
+        if set_key:
+            for step in grp["steps"]:
+                step["key"] = self.build_key(
+                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                )
+        return self.add_step(grp, depends_on_build=depends_on_build)
 
     def to_dict(self):
         """Render the pipeline as a dictionary."""
@@ -327,7 +347,9 @@ def to_json(self):
     def devtool_test(self, devtool_opts=None, pytest_opts=None):
         """Generate a `devtool test` command"""
         cmds = []
-        parts = ["./tools/devtool -y test", "--no-build"]
+        parts = ["./tools/devtool -y test"]
+        if self.shared_build is not None:
+            parts.append("--no-build")
         if devtool_opts:
             parts.append(devtool_opts)
         parts.append("--")

.buildkite/pipeline_cpu_template.py

@@ -23,7 +23,7 @@ class BkStep(str, Enum):
 cpu_template_test = {
     "rdmsr": {
         BkStep.COMMAND: [
-            "tools/devtool -y test --no-build -- -s -ra -m nonci -n4 --log-cli-level=INFO integration_tests/functional/test_cpu_features.py -k 'test_cpu_rdmsr' "
+            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_rdmsr' "
         ],
         BkStep.LABEL: "📖 rdmsr",
         "instances": ["c5n.metal", "m5n.metal", "m6a.metal", "m6i.metal"],
@@ -34,13 +34,13 @@ class BkStep(str, Enum):
             "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
         ],
         BkStep.LABEL: "🖐️ fingerprint",
-        "instances": DEFAULT_INSTANCES,
+        "instances": DEFAULT_INSTANCES.keys(),
         "platforms": DEFAULT_PLATFORMS,
     },
     "cpuid_wrmsr": {
         "snapshot": {
             BkStep.COMMAND: [
-                "tools/devtool -y test --no-build -- -s -ra -m nonci -n4 --log-cli-level=INFO integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
                 "mkdir -pv tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
                 "sudo mv tests/snapshot_artifacts/* tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
             ],
@@ -52,7 +52,7 @@ class BkStep(str, Enum):
             BkStep.COMMAND: [
                 "buildkite-agent artifact download tests/snapshot_artifacts_upload/{instance}_{os}_{kv}/**/* .",
                 "mv tests/snapshot_artifacts_upload/{instance}_{os}_{kv} tests/snapshot_artifacts",
-                "tools/devtool -y test --no-build -- -s -ra -m nonci -n4 --log-cli-level=INFO integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
             ],
             BkStep.LABEL: "📸 load snapshot artifacts created on {instance} {snapshot_os} {snapshot_kv} to {restore_instance} {restore_os} {restore_kv}",
             BkStep.TIMEOUT: 30,
@@ -117,11 +117,11 @@ def group_snapshot_restore(test_step):
                 BkStep.COMMAND: restore_commands,
                 BkStep.LABEL: restore_label,
                 BkStep.TIMEOUT: test_step["restore"][BkStep.TIMEOUT],
-                "agents": [
-                    f"instance={restore_instance}",
-                    f"kv={restore_kv}",
-                    f"os={restore_os}",
-                ],
+                "agents": {
+                    "instance": restore_instance,
+                    "kv": restore_kv,
+                    "os": restore_os,
+                },
             }
         )
 

.buildkite/pipeline_perf.py

@@ -120,8 +120,8 @@
 # will pin steps running on instances "m6i.metal" with kernel version tagged "linux_6.1"
 # to a new kernel version tagged "linux_6.1-pinned"
 pins = {
+    # TODO: Unpin when performance instability on m6i/5.10 has gone.
     "linux_5.10-pinned": {"instance": "m6i.metal", "kv": "linux_5.10"},
-    "linux_6.1-pinned": {"instance": "m6i.metal", "kv": "linux_6.1"},
 }
 
 

.buildkite/pipeline_pr.py

@@ -14,18 +14,23 @@
     "agents": {"ag": 1},
 }
 
+changed_files = get_changed_files()
+DOC_ONLY_CHANGE = False
+if changed_files and all(f.suffix == ".md" for f in changed_files):
+    DOC_ONLY_CHANGE = True
 pipeline = BKPipeline(
     priority=DEFAULT_PRIORITY,
     timeout_in_minutes=45,
-    initial_steps=[
-        {
-            "command": "./tools/devtool -y checkstyle",
-            "label": "🪶 Style",
-        },
-    ],
+    with_build_step=not DOC_ONLY_CHANGE,
 )
 
-changed_files = get_changed_files()
+pipeline.add_step(
+    {
+        "command": "./tools/devtool -y checkstyle",
+        "label": "🪶 Style",
+    },
+    depends_on_build=False,
+)
 
 # run sanity build of devtool if Dockerfile is changed
 if any(x.parent.name == "devctr" for x in changed_files):
@@ -48,13 +53,14 @@
 ):
     kani_grp = pipeline.build_group(
         "🔍 Kani",
-        "./tools/devtool -y test -- ../tests/integration_tests/test_kani.py -n auto",
+        "./tools/devtool -y test --no-build -- ../tests/integration_tests/test_kani.py -n auto",
         # Kani step default
         # Kani runs fastest on m6a.metal
-        instances=["m6a.metal"],
+        instances=["m6a.metal", "m7g.metal"],
         platforms=[("al2", "linux_5.10")],
         timeout_in_minutes=300,
         **DEFAULTS_PERF,
+        depends_on_build=False,
     )
     # modify Kani steps' label
     for step in kani_grp["steps"]:
@@ -64,6 +70,7 @@
     pipeline.build_group(
         "📦 Build",
         pipeline.devtool_test(pytest_opts="integration_tests/build/"),
+        depends_on_build=False,
     )
 
     pipeline.build_group(

.buildkite/pipeline_pr_no_block.py

@@ -11,6 +11,7 @@
 DEFAULT_PRIORITY = 1
 
 pipeline = BKPipeline(
+    with_build_step=False,
     timeout_in_minutes=45,
     # some non-blocking tests are performance, so make sure they get ag=1 instances
     priority=DEFAULT_PRIORITY + 1,

.dockerignore

@@ -3,3 +3,6 @@ build/
 src/
 tests/
 docs/
+resources/
+tools/test-popular-containers/
+test_results/

.github/CODEOWNERS

@@ -1,19 +1,19 @@
 # All markdown files
-*.md @xmarcalx @kalyazin @pb8o
+*.md @xmarcalx @kalyazin @pb8o @Manciukic
 
 # But not the ones in docs/
 docs/*.md
 
 # Except these specific ones
-docs/getting-started.md @xmarcalx @kalyazin @pb8o
-docs/prod-host-setup.md @xmarcalx @kalyazin @pb8o
+docs/getting-started.md @xmarcalx @kalyazin @pb8o @Manciukic
+docs/prod-host-setup.md @xmarcalx @kalyazin @pb8o @Manciukic
 
 # Also cover all "*policy*.md" documents
-**/*policy*.md @xmarcalx @kalyazin @pb8o
-**/*POLICY*.md @xmarcalx @kalyazin @pb8o
+**/*policy*.md @xmarcalx @kalyazin @pb8o @Manciukic
+**/*POLICY*.md @xmarcalx @kalyazin @pb8o @Manciukic
 
 # Also these non-md files in the repository root
-THIRD_PARTY @xmarcalx @kalyazin @pb8o
-LICENSE @xmarcalx @kalyazin @pb8o
-NOTICE @xmarcalx @kalyazin @pb8o
-PGP-KEY.asc @xmarcalx @kalyazin @pb8o
+THIRD_PARTY @xmarcalx @kalyazin @pb8o @Manciukic
+LICENSE @xmarcalx @kalyazin @pb8o @Manciukic
+NOTICE @xmarcalx @kalyazin @pb8o @Manciukic
+PGP-KEY.asc @xmarcalx @kalyazin @pb8o @Manciukic

.gitignore

@@ -12,3 +12,6 @@ test_results/*
 *.profraw
 .DS_Store
 *.bin
+/resources/linux
+/resources/x86_64
+/resources/aarch64

.mailmap

@@ -18,12 +18,18 @@ Alexandru Cihodaru <cihodar@amazon.com>
 Liviu Berciu <lberciu@amazon.com>
 Jonathan Woollett-Light <jcawl@amazon.co.uk> <jonathanwoollettlight@gmail.com>
 Jonathan Woollett-Light <jcawl@amazon.co.uk> <jonthanwoollettlight@gmail.com>
-Sudan Landge <sudanl@amazon.com> <sudan@amazon.co.uk>
+Sudan Landge <sudanl@amazon.co.uk> <119602619+sudanl0@users.noreply.github.com>
+Sudan Landge <sudanl@amazon.co.uk> <sudanl@amazon.com>
 karthik nedunchezhiyan <karthik.n@zohocorp.com>
 Babis Chalios <bchalios@amazon.es> <babis.chalios@gmail.com>
 Pablo Barbáchano <pablob@amazon.com>
 Nikita Kalyazin <kalyazin@amazon.co.uk> <kalyazin@amazon.com>
 Trăistaru Andrei Cristian <atc@amazon.com>
 Trăistaru Andrei Cristian <atc@amazon.com> <56828222+andreitraistaru@users.noreply.github.com>
-Sudan Landge <sudanl@amazon.com> <sudanl@amazon.co.uk>
 Takahiro Itazuri <itazur@amazon.com> <zulinx86@gmail.com>
+Jack Thomson <jackabt@amazon.co.uk> <jackabt@amazon.com>
+Ashwin Ginoria <aginoria@amazon.com> <ec2-user@ip-10-0-10-88.us-west-2.compute.internal>
+Muskaan Singla <msinglaa@amazon.com> <ec2-user@ip-172-31-1-28.us-west-2.compute.internal>
+Egor Lazarchuk <yegorlz@amazon.co.uk>
+Nikita Zakirov <zakironi@amazon.com> <ec2-user@ip-10-0-15-219.us-west-2.compute.internal>
+Tomoya Iwata <iwata.tomoya@classmethod.jp>

.python-version

@@ -0,0 +1 @@
+3.10.14