Support PEP420 (implicit namespace packages) as --pyargs target. (pytest-dev#13426)

Previously, when running `--pyargs pkg`, if you didn't have `pkg/__init__.py`, pytest would fail with `ERROR: module or package not found: pkg (missing __init__.py?)`.

If used in conjunction with `consider_namespace_packages` in config, pytest discovers the package and tests inside it correctly.

If used in conjunction with `consider_namespace_packages` in config, test
modules get correct `__package__` and `__name__` attributes as well.

In addition, remove `"namespace"` origin handling -- this value isn't used since python 3.8.
See:
- python/cpython#5481
- https://docs.python.org/3/library/importlib.html#importlib.machinery.ModuleSpec.submodule_search_locations

Fixes pytest-dev#478
Fixes pytest-dev#2371
Fixes pytest-dev#10569

Author: karlicoss

changelog/478.feature.rst

@@ -0,0 +1,3 @@
+Support PEP420 (implicit namespace packages) as `--pyargs` target when :confval:`consider_namespace_packages` is `true` in the config.
+
+Previously, this option only impacted package names, now it also impacts tests discovery.

doc/en/reference/reference.rst

@@ -1384,6 +1384,7 @@ passed multiple times. The expected format is ``name=value``. For example::
    when collecting Python modules. Default is ``False``.
 
    Set to ``True`` if the package you are testing is part of a namespace package.
+   Namespace packages are also supported as ``--pyargs`` target.
 
    Only `native namespace packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#native-namespace-packages>`__
    are supported, with no plans to support `legacy namespace packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages/#legacy-namespace-packages>`__.

src/_pytest/main.py

@@ -774,6 +774,9 @@ def perform_collect(
         self._collection_cache = {}
         self.items = []
         items: Sequence[nodes.Item | nodes.Collector] = self.items
+        consider_namespace_packages: bool = self.config.getini(
+            "consider_namespace_packages"
+        )
         try:
             initialpaths: list[Path] = []
             initialpaths_with_parents: list[Path] = []
@@ -782,6 +785,7 @@ def perform_collect(
                     self.config.invocation_params.dir,
                     arg,
                     as_pypath=self.config.option.pyargs,
+                    consider_namespace_packages=consider_namespace_packages,
                 )
                 self._initial_parts.append(collection_argument)
                 initialpaths.append(collection_argument.path)
@@ -981,7 +985,9 @@ def genitems(self, node: nodes.Item | nodes.Collector) -> Iterator[nodes.Item]:
                 node.ihook.pytest_collectreport(report=rep)
 
 
-def search_pypath(module_name: str) -> str | None:
+def search_pypath(
+    module_name: str, *, consider_namespace_packages: bool = False
+) -> str | None:
     """Search sys.path for the given a dotted module name, and return its file
     system path if found."""
     try:
@@ -991,13 +997,29 @@ def search_pypath(module_name: str) -> str | None:
     # ValueError: not a module name
     except (AttributeError, ImportError, ValueError):
         return None
-    if spec is None or spec.origin is None or spec.origin == "namespace":
+
+    if spec is None:
         return None
-    elif spec.submodule_search_locations:
-        return os.path.dirname(spec.origin)
-    else:
+
+    if (
+        spec.submodule_search_locations is None
+        or len(spec.submodule_search_locations) == 0
+    ):
+        # Must be a simple module.
         return spec.origin
 
+    if consider_namespace_packages:
+        # If submodule_search_locations is set, it's a package (regular or namespace).
+        # Typically there is a single entry, but documentation claims it can be empty too
+        #  (e.g. if the package has no physical location).
+        return spec.submodule_search_locations[0]
+
+    if spec.origin is None:
+        # This is only the case for namespace packages
+        return None
+
+    return os.path.dirname(spec.origin)
+
 
 @dataclasses.dataclass(frozen=True)
 class CollectionArgument:
@@ -1009,7 +1031,11 @@ class CollectionArgument:
 
 
 def resolve_collection_argument(
-    invocation_path: Path, arg: str, *, as_pypath: bool = False
+    invocation_path: Path,
+    arg: str,
+    *,
+    as_pypath: bool = False,
+    consider_namespace_packages: bool = False,
 ) -> CollectionArgument:
     """Parse path arguments optionally containing selection parts and return (fspath, names).
 
@@ -1049,7 +1075,9 @@ def resolve_collection_argument(
         parts[-1] = f"{parts[-1]}{squacket}{rest}"
     module_name = None
     if as_pypath:
-        pyarg_strpath = search_pypath(strpath)
+        pyarg_strpath = search_pypath(
+            strpath, consider_namespace_packages=consider_namespace_packages
+        )
         if pyarg_strpath is not None:
             module_name = strpath
             strpath = pyarg_strpath

testing/test_collection.py

@@ -1935,3 +1935,64 @@ def test_func():
     result = pytester.runpytest()
     assert result.ret == 0
     result.stdout.fnmatch_lines(["*1 passed*"])
+
+
+@pytest.mark.parametrize("import_mode", ["prepend", "importlib", "append"])
+def test_namespace_packages(pytester: Pytester, import_mode: str):
+    pytester.makeini(
+        f"""
+        [pytest]
+        consider_namespace_packages = true
+        pythonpath = .
+        python_files = *.py
+        addopts = --import-mode {import_mode}
+    """
+    )
+    pytester.makepyfile(
+        **{
+            "pkg/module1.py": "def test_module1(): pass",
+            "pkg/subpkg_namespace/module2.py": "def test_module1(): pass",
+            "pkg/subpkg_regular/__init__.py": "",
+            "pkg/subpkg_regular/module3": "def test_module3(): pass",
+        }
+    )
+
+    # should collect when called with top-level package correctly
+    result = pytester.runpytest("--collect-only", "--pyargs", "pkg")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 3 items",
+            "<Dir pkg>",
+            "  <Module module1.py>",
+            "    <Function test_module1>",
+            "  <Dir subpkg_namespace>",
+            "    <Module module2.py>",
+            "      <Function test_module1>",
+            "  <Package subpkg_regular>",
+            "    <Module module3.py>",
+            "      <Function test_module3>",
+        ]
+    )
+
+    # should also work when called against a more specific subpackage/module
+    result = pytester.runpytest("--collect-only", "--pyargs", "pkg.subpkg_namespace")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 1 item",
+            "<Dir pkg>",
+            "  <Dir subpkg_namespace>",
+            "    <Module module2.py>",
+            "      <Function test_module1>",
+        ]
+    )
+
+    result = pytester.runpytest("--collect-only", "--pyargs", "pkg.subpkg_regular")
+    result.stdout.fnmatch_lines(
+        [
+            "collected 1 item",
+            "<Dir pkg>",
+            "  <Package subpkg_regular>",
+            "    <Module module3.py>",
+            "      <Function test_module3>",
+        ]
+    )

testing/test_main.py

@@ -169,8 +169,13 @@ def test_dir(self, invocation_path: Path) -> None:
         ):
             resolve_collection_argument(invocation_path, "src/pkg::foo::bar")
 
-    def test_pypath(self, invocation_path: Path) -> None:
+    @pytest.mark.parametrize("namespace_package", [False, True])
+    def test_pypath(self, namespace_package: bool, invocation_path: Path) -> None:
         """Dotted name and parts."""
+        if namespace_package:
+            # Namespace package doesn't have to contain __init__py
+            (invocation_path / "src/pkg/__init__.py").unlink()
+
         assert resolve_collection_argument(
             invocation_path, "pkg.test", as_pypath=True
         ) == CollectionArgument(
@@ -186,7 +191,10 @@ def test_pypath(self, invocation_path: Path) -> None:
             module_name="pkg.test",
         )
         assert resolve_collection_argument(
-            invocation_path, "pkg", as_pypath=True
+            invocation_path,
+            "pkg",
+            as_pypath=True,
+            consider_namespace_packages=namespace_package,
         ) == CollectionArgument(
             path=invocation_path / "src/pkg",
             parts=[],
@@ -197,7 +205,10 @@ def test_pypath(self, invocation_path: Path) -> None:
             UsageError, match=r"package argument cannot contain :: selection parts"
         ):
             resolve_collection_argument(
-                invocation_path, "pkg::foo::bar", as_pypath=True
+                invocation_path,
+                "pkg::foo::bar",
+                as_pypath=True,
+                consider_namespace_packages=namespace_package,
             )
 
     def test_parametrized_name_with_colons(self, invocation_path: Path) -> None: