Improve integration test options

.github/workflows/test.yml

@@ -81,7 +81,7 @@ jobs:
           CIBW_ENABLE=all
         else
           # get the default CIBW_ENABLE value from the test module
-          CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.conftest as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)')
+          CIBW_ENABLE=$(uv run --no-sync python -c 'import sys, test.utils as c; sys.stdout.write(c.DEFAULT_CIBW_ENABLE)')
 
           # if this is a PR, check for labels
           if [[ -n "$GITHUB_PR_LABEL_CI_PYPY" ]]; then

docs/contributing.md

@@ -78,11 +78,11 @@ nox -s tests -- test -k before_build
 
 A few notes-
 
-- Because they run inside a container, Linux tests can run on all platforms where Docker is installed, so they're convenient for running integration tests locally. Set CIBW_PLATFORM to do this: `CIBW_PLATFORM=linux nox -s tests -- test`.
+- Because they run inside a container, Linux tests can run on all platforms where Docker is installed, so they're convenient for running integration tests locally. Set the `--platform` flag on pytest to do this: `nox -s tests -- test --platform linux`.
 
 - Running the macOS integration tests requires _system installs_ of Python from python.org for all the versions that are tested. We won't attempt to install these when running locally, but you can do so manually using the URL in the error message that is printed when the install is not found.
 
-- The 'enable groups' run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by setting the [CIBW_ENABLE](options.md#enable) environment variable. On GitHub PRs, you can add a label to the PR to enable these groups.
+- The ['enable groups'](options.md#enable) run by default are just 'cpython-prerelease' and 'cpython-freethreading'. You can add other groups like pypy or graalpy by passing the `--enable` argument to pytest, i.e. `nox -s tests -- test --enable pypy`. On GitHub PRs, you can add a label to the PR to enable these groups.
 
 #### Running pytest directly
 

test/conftest.py

@@ -9,12 +9,11 @@
 from cibuildwheel.architecture import Architecture
 from cibuildwheel.ci import detect_ci_provider
 from cibuildwheel.options import CommandLineArguments, Options
+from cibuildwheel.selector import EnableGroup
+from cibuildwheel.typing import PLATFORMS
 from cibuildwheel.venv import find_uv
 
-from .utils import EMULATED_ARCHS, platform
-
-# default to just cpython
-DEFAULT_CIBW_ENABLE = "cpython-freethreading cpython-prerelease cpython-experimental-riscv64"
+from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_platform
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
@@ -32,8 +31,47 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=False,
         help="macOS cp38 uses the universal2 installer",
     )
+    parser.addoption(
+        "--enable",
+        action="store",
+        default=None,
+        help="Set the CIBW_ENABLE environment variable for all tests.",
+    )
+    parser.addoption(
+        "--platform",
+        action="store",
+        default=None,
+        help="Set the CIBW_PLATFORM environment variable for all tests.",
+    )
+
+
+def pytest_configure(config):
+    flag_enable = config.getoption("--enable")
+    flag_platform = config.getoption("--platform")
+
+    if flag_enable is not None and "CIBW_ENABLE" in os.environ:
+        msg = (
+            "Both --enable pytest option and CIBW_ENABLE environment variable are set. "
+            "Please specify only one."
+        )
+        raise pytest.UsageError(msg)
+    if flag_platform is not None and "CIBW_PLATFORM" in os.environ:
+        msg = (
+            "Both --platform pytest option and CIBW_PLATFORM environment variable are set. "
+            "Please specify only one."
+        )
+        raise pytest.UsageError(msg)
+
+    if flag_enable is not None:
+        EnableGroup.parse_option_value(flag_enable)
+        os.environ["CIBW_ENABLE"] = flag_enable
+    if flag_enable is None and "CIBW_ENABLE" not in os.environ:
+        # Set default value for CIBW_ENABLE
+        os.environ["CIBW_ENABLE"] = DEFAULT_CIBW_ENABLE
 
-    os.environ.setdefault("CIBW_ENABLE", DEFAULT_CIBW_ENABLE)
+    if flag_platform is not None:
+        assert flag_platform in PLATFORMS, f"Invalid platform: {flag_platform}"
+        os.environ["CIBW_PLATFORM"] = flag_platform
 
 
 def docker_warmup(request: pytest.FixtureRequest) -> None:
@@ -91,7 +129,7 @@ def docker_warmup_fixture(
     request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str
 ) -> None:
     # if we're in CI testing linux, let's warm-up docker images
-    if detect_ci_provider() is None or platform != "linux":
+    if detect_ci_provider() is None or get_platform() != "linux":
         return None
     if request.config.getoption("--run-emulation", default=None) is not None:
         # emulation tests only run one test in CI, caching the image only slows down the test
@@ -116,7 +154,7 @@ def docker_warmup_fixture(
 @pytest.fixture(params=["pip", "build"])
 def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]:
     frontend = request.param
-    if platform == "pyodide" and frontend == "pip":
+    if get_platform() == "pyodide" and frontend == "pip":
         pytest.skip("Can't use pip as build frontend for pyodide platform")
 
     return {"CIBW_BUILD_FRONTEND": frontend}
@@ -125,7 +163,7 @@ def build_frontend_env_nouv(request: pytest.FixtureRequest) -> dict[str, str]:
 @pytest.fixture
 def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str]:
     frontend = build_frontend_env_nouv["CIBW_BUILD_FRONTEND"]
-    if frontend != "build" or platform == "pyodide" or find_uv() is None:
+    if frontend != "build" or get_platform() == "pyodide" or find_uv() is None:
         return build_frontend_env_nouv
 
     return {"CIBW_BUILD_FRONTEND": "build[uv]"}
@@ -134,7 +172,7 @@ def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str
 @pytest.fixture
 def docker_cleanup() -> Generator[None, None, None]:
     def get_images() -> set[str]:
-        if detect_ci_provider() is None or platform != "linux":
+        if detect_ci_provider() is None or get_platform() != "linux":
             return set()
         images = subprocess.run(
             ["docker", "image", "ls", "--format", "{{json .ID}}"],

test/test_0_basic.py

@@ -1,4 +1,3 @@
-import os
 import textwrap
 
 import pytest
@@ -40,7 +39,7 @@ def test(tmp_path, build_frontend_env, capfd):
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
     assert set(actual_wheels) == set(expected_wheels)
 
-    enable_groups = EnableGroup.parse_option_value(os.environ.get("CIBW_ENABLE", ""))
+    enable_groups = utils.get_enable_groups()
     if EnableGroup.GraalPy not in enable_groups:
         # Verify pip warning not shown
         captured = capfd.readouterr()

test/test_abi_variants.py

@@ -47,7 +47,7 @@ def test_abi3(tmp_path):
     )
 
     # check that the expected wheels are produced
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         # there's only 1 possible configuration for pyodide, cp312
         expected_wheels = utils.expected_wheels("spam", "0.1.0", python_abi_tags=["cp310-abi3"])
     else:
@@ -196,7 +196,7 @@ def test_abi_none(tmp_path, capfd):
     # check that each wheel was built once, and reused
     captured = capfd.readouterr()
     assert "Building wheel..." in captured.out
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         # there's only 1 possible configuration for pyodide, we won't see the message expected on following builds
         assert "Found previously built wheel" not in captured.out
     else:

test/test_before_build.py

@@ -6,7 +6,7 @@
 from . import test_projects, utils
 
 # pyodide does not support building without isolation, need to check the base_prefix
-SYS_PREFIX = f"sys.{'base_' if utils.platform == 'pyodide' else ''}prefix"
+SYS_PREFIX = f"sys.{'base_' if utils.get_platform() == 'pyodide' else ''}prefix"
 
 
 project_with_before_build_asserts = test_projects.new_c_project(
@@ -45,7 +45,7 @@ def test(tmp_path):
         f'''python -c "import pathlib, sys; pathlib.Path('{{project}}/pythonprefix_bb.txt').write_text({SYS_PREFIX})"'''
     )
     frontend = "build"
-    if utils.platform != "pyodide":
+    if utils.get_platform() != "pyodide":
         before_build = f"python -m pip install setuptools && {before_build}"
         frontend = f"{frontend};args: --no-isolation"
 

test/test_before_test.py

@@ -43,7 +43,7 @@ def test(tmp_path, build_frontend_env):
         '''python -c "import pathlib, sys; pathlib.Path('{project}/pythonprefix_bt.txt').write_text(sys.prefix)"''',
     ]
 
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         before_test_steps.extend(
             ["pyodide build {project}/dependency", "pip install --find-links dist/ spam"]
         )

test/test_build_frontend_args.py

@@ -20,7 +20,7 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
 
     # the build will fail because the frontend is called with '-h' - it prints the help message
     add_env = {"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"}
-    if utils.platform == "pyodide":
+    if utils.get_platform() == "pyodide":
         add_env["TERM"] = "dumb"  # disable color / style
     with pytest.raises(subprocess.CalledProcessError):
         utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True)
@@ -32,7 +32,7 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
     if frontend_name == "pip":
         assert "Usage:" in captured.out
         assert "Wheel Options:" in captured.out
-    elif utils.platform == "pyodide":
+    elif utils.get_platform() == "pyodide":
         assert "Usage: pyodide build" in captured.out
     else:
         assert "usage:" in captured.out

test/test_container_engine.py

@@ -6,7 +6,7 @@
 
 
 def test_podman(tmp_path, capfd, request):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
     if not request.config.getoption("--run-podman"):
@@ -36,7 +36,7 @@ def test_podman(tmp_path, capfd, request):
 
 
 def test_create_args(tmp_path, capfd):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
     project_dir = tmp_path / "project"

test/test_container_images.py

@@ -23,7 +23,7 @@
 
 @pytest.mark.usefixtures("docker_cleanup")
 def test(tmp_path):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
     if platform.machine() not in ["x86_64", "i686"]:
         pytest.skip(

test/test_cpp_standards.py

@@ -60,7 +60,7 @@ def test_cpp11(tmp_path):
     project_dir = tmp_path / "project"
     cpp11_project = cpp_test_project.copy()
     cpp11_project.template_context["extra_compile_args"] = (
-        ["/std:c++11"] if utils.platform == "windows" else ["-std=c++11"]
+        ["/std:c++11"] if utils.get_platform() == "windows" else ["-std=c++11"]
     )
     cpp11_project.template_context["spam_cpp_top_level_add"] = "#include <array>"
     cpp11_project.generate(project_dir)
@@ -76,7 +76,7 @@ def test_cpp14(tmp_path):
     project_dir = tmp_path / "project"
     cpp14_project = cpp_test_project.copy()
     cpp14_project.template_context["extra_compile_args"] = (
-        ["/std:c++14"] if utils.platform == "windows" else ["-std=c++14"]
+        ["/std:c++14"] if utils.get_platform() == "windows" else ["-std=c++14"]
     )
     cpp14_project.template_context["spam_cpp_top_level_add"] = "int a = 100'000;"
     cpp14_project.generate(project_dir)
@@ -92,7 +92,7 @@ def test_cpp17(tmp_path):
     project_dir = tmp_path / "project"
     cpp17_project = cpp_test_project.copy()
     cpp17_project.template_context["extra_compile_args"] = [
-        "/std:c++17" if utils.platform == "windows" else "-std=c++17"
+        "/std:c++17" if utils.get_platform() == "windows" else "-std=c++17"
     ]
     cpp17_project.template_context["spam_cpp_top_level_add"] = r"""
     #include <utility>
@@ -104,7 +104,7 @@ def test_cpp17(tmp_path):
         pytest.skip("Visual Studio 2015 does not support C++17")
 
     add_env = {}
-    if utils.platform == "macos":
+    if utils.get_platform() == "macos":
         add_env["MACOSX_DEPLOYMENT_TARGET"] = "10.13"
 
     actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True)

test/test_dependency_versions.py

@@ -53,11 +53,15 @@ def get_versions_from_constraint_file(constraint_file: Path) -> dict[str, str]:
 
 @pytest.mark.parametrize("python_version", ["3.8", "3.12"])
 def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv):
-    if utils.platform == "linux":
+    if utils.get_platform() == "linux":
         pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")
-    if python_version != "3.12" and utils.platform == "pyodide":
+    if python_version != "3.12" and utils.get_platform() == "pyodide":
         pytest.skip(f"pyodide does not support Python {python_version}")
-    if python_version == "3.8" and utils.platform == "windows" and platform.machine() == "ARM64":
+    if (
+        python_version == "3.8"
+        and utils.get_platform() == "windows"
+        and platform.machine() == "ARM64"
+    ):
         pytest.skip(f"Windows ARM64 does not support Python {python_version}")
 
     project_dir = tmp_path / "project"
@@ -96,7 +100,7 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv):
 
 @pytest.mark.parametrize("method", ["inline", "file"])
 def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv):
-    if utils.platform == "linux":
+    if utils.get_platform() == "linux":
         pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")
 
     project_dir = tmp_path / "project"
@@ -129,7 +133,7 @@ def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv):
     build_environment = {}
 
     if (
-        utils.platform == "windows"
+        utils.get_platform() == "windows"
         and method == "file"
         and build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build"
     ):
@@ -159,7 +163,7 @@ def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv):
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
 
     if (
-        utils.platform == "windows"
+        utils.get_platform() == "windows"
         and method == "file"
         and build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build"
     ):

test/test_emulation.py

@@ -53,7 +53,7 @@ def test(tmp_path, request):
 
 
 def test_setting_arch_on_other_platforms(tmp_path, capfd):
-    if utils.platform == "linux":
+    if utils.get_platform() == "linux":
         pytest.skip("this test checks the behaviour on platforms other than linux")
 
     project_dir = tmp_path / "project"

test/test_environment.py

@@ -64,7 +64,7 @@ def test_overridden_path(tmp_path, capfd):
     output_dir.mkdir()
 
     # mess up PATH, somehow
-    if utils.platform == "linux":
+    if utils.get_platform() == "linux":
         with pytest.raises(subprocess.CalledProcessError):
             utils.cibuildwheel_run(
                 project_dir,
@@ -128,7 +128,7 @@ def test_overridden_pip_constraint(tmp_path, build_frontend):
     )
     project.generate(project_dir)
 
-    if utils.platform == "linux":
+    if utils.get_platform() == "linux":
         # put the constraints file in the project directory, so it's available
         # in the docker container
         constraints_file = project_dir / "constraints.txt"

test/test_ios.py

@@ -38,7 +38,7 @@ def test_platform(self):
     ],
 )
 def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
-    if utils.platform != "macos":
+    if utils.get_platform() != "macos":
         pytest.skip("this test can only run on macOS")
     if utils.get_xcode_version() < (13, 0):
         pytest.skip("this test only works with Xcode 13.0 or greater")
@@ -103,7 +103,7 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
 
 def test_no_test_sources(tmp_path, capfd):
     """Build will fail if test-sources isn't defined."""
-    if utils.platform != "macos":
+    if utils.get_platform() != "macos":
         pytest.skip("this test can only run on macOS")
     if utils.get_xcode_version() < (13, 0):
         pytest.skip("this test only works with Xcode 13.0 or greater")
@@ -130,7 +130,7 @@ def test_no_test_sources(tmp_path, capfd):
 
 def test_missing_xbuild_tool(tmp_path, capfd):
     """Build will fail if xbuild-tools references a non-existent tool."""
-    if utils.platform != "macos":
+    if utils.get_platform() != "macos":
         pytest.skip("this test can only run on macOS")
     if utils.get_xcode_version() < (13, 0):
         pytest.skip("this test only works with Xcode 13.0 or greater")
@@ -158,7 +158,7 @@ def test_missing_xbuild_tool(tmp_path, capfd):
 
 def test_no_xbuild_tool_definition(tmp_path, capfd):
     """Build will succeed with a warning if there is no xbuild-tools definition."""
-    if utils.platform != "macos":
+    if utils.get_platform() != "macos":
         pytest.skip("this test can only run on macOS")
     if utils.get_xcode_version() < (13, 0):
         pytest.skip("this test only works with Xcode 13.0 or greater")
@@ -203,7 +203,7 @@ def test_no_xbuild_tool_definition(tmp_path, capfd):
 
 def test_empty_xbuild_tool_definition(tmp_path, capfd):
     """Build will succeed with no warning if there is an empty xbuild-tools definition."""
-    if utils.platform != "macos":
+    if utils.get_platform() != "macos":
         pytest.skip("this test can only run on macOS")
     if utils.get_xcode_version() < (13, 0):
         pytest.skip("this test only works with Xcode 13.0 or greater")

test/test_linux_python.py

@@ -7,7 +7,7 @@
 
 
 def test_python_exist(tmp_path, capfd):
-    if utils.platform != "linux":
+    if utils.get_platform() != "linux":
         pytest.skip("the test is only relevant to the linux build")
     machine = platform.machine()
     if machine not in ["x86_64", "i686"]: