From 7bbb864443eead6fed982cc236cf137d638cc967 Mon Sep 17 00:00:00 2001 From: Jack O'Connor Date: Thu, 23 Oct 2025 10:10:10 -0700 Subject: [PATCH] [ty] implement `typing.NewType` by adding `Type::NewTypeInstance` --- crates/ty/docs/rules.md | 164 +++++--- crates/ty_ide/src/completion.rs | 3 +- crates/ty_ide/src/goto.rs | 15 +- .../resources/mdtest/annotations/new_types.md | 382 +++++++++++++++++- .../resources/mdtest/class/super.md | 6 +- ...NewTy\342\200\246_(9847ea9eddc316b4).snap" | 58 +++ ...bclass_a\342\200\246_(fd3c73e2a9f04).snap" | 37 ++ ...Objec\342\200\246_(b753048091f275c0).snap" | 152 +++---- crates/ty_python_semantic/src/types.rs | 151 ++++++- .../src/types/bound_super.rs | 3 + crates/ty_python_semantic/src/types/class.rs | 8 + .../src/types/class_base.rs | 12 +- .../src/types/definition.rs | 7 +- .../src/types/diagnostic.rs | 44 ++ .../ty_python_semantic/src/types/display.rs | 1 + .../ty_python_semantic/src/types/function.rs | 5 + .../src/types/ide_support.rs | 4 + .../src/types/infer/builder.rs | 176 +++++++- .../types/infer/builder/type_expression.rs | 10 + crates/ty_python_semantic/src/types/narrow.rs | 3 +- .../ty_python_semantic/src/types/newtype.rs | 266 ++++++++++++ .../src/types/type_ordering.rs | 4 + .../ty_python_semantic/src/types/visitor.rs | 12 + .../e2e__commands__debug_command.snap | 1 + ty.schema.json | 10 + 25 files changed, 1343 insertions(+), 191 deletions(-) create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_The_base_of_a_`NewTy\342\200\246_(9847ea9eddc316b4).snap" create mode 100644 "crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_Trying_to_subclass_a\342\200\246_(fd3c73e2a9f04).snap" create mode 100644 crates/ty_python_semantic/src/types/newtype.rs diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index 951c364462627..58b1db584c113 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -39,7 +39,7 @@ def test(): -> "int": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -63,7 +63,7 @@ Calling a non-callable object will raise a `TypeError` at runtime. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -95,7 +95,7 @@ f(int) # error Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -126,7 +126,7 @@ a = 1 Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -158,7 +158,7 @@ class C(A, B): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -190,7 +190,7 @@ class B(A): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -217,7 +217,7 @@ class B(A, A): ... Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -329,7 +329,7 @@ def test(): -> "Literal[5]": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -359,7 +359,7 @@ class C(A, B): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -385,7 +385,7 @@ t[3] # IndexError: tuple index out of range Default level: error · Added in 0.0.1-alpha.12 · Related issues · -View source +View source @@ -474,7 +474,7 @@ an atypical memory layout. Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -501,7 +501,7 @@ func("foo") # error: [invalid-argument-type] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -529,7 +529,7 @@ a: int = '' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -563,7 +563,7 @@ C.instance_var = 3 # error: Cannot assign to instance variable Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -599,7 +599,7 @@ asyncio.run(main()) Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -623,7 +623,7 @@ class A(42): ... # error: [invalid-base] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -650,7 +650,7 @@ with 1: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -679,7 +679,7 @@ a: str Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -723,7 +723,7 @@ except ZeroDivisionError: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -756,7 +756,7 @@ class C[U](Generic[T]): ... Default level: error · Added in 0.0.1-alpha.17 · Related issues · -View source +View source @@ -795,7 +795,7 @@ carol = Person(name="Carol", age=25) # typo! Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -830,7 +830,7 @@ def f(t: TypeVar("U")): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -864,7 +864,7 @@ class B(metaclass=f): ... Default level: error · Added in 0.0.1-alpha.19 · Related issues · -View source +View source @@ -890,13 +890,43 @@ in a class's bases list. TypeError: can only inherit from a NamedTuple type and Generic ``` +## `invalid-newtype` + + +Default level: error · +Preview (since 1.0.0) · +Related issues · +View source + + + +**What it does** + +Checks for the creation of invalid `NewType`s + +**Why is this bad?** + +There are several requirements that you must follow when creating a `NewType`. + +**Examples** + +```python +from typing import NewType + +def get_name() -> str: ... + +Foo = NewType("Foo", int) # okay +Bar = NewType(get_name(), int) # error: The first argument to `NewType` must be a string literal +Baz = NewType("Baz", int | str) # error: invalid base for `typing.NewType` +``` + ## `invalid-overload` Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -946,7 +976,7 @@ def foo(x: int) -> int: ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -972,7 +1002,7 @@ def f(a: int = ''): ... Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1003,7 +1033,7 @@ P2 = ParamSpec("S2") # error: ParamSpec name must match the variable it's assig Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1037,7 +1067,7 @@ TypeError: Protocols can only inherit from other protocols, got Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1086,7 +1116,7 @@ def g(): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1111,7 +1141,7 @@ def func() -> int: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1169,7 +1199,7 @@ TODO #14889 Default level: error · Added in 0.0.1-alpha.6 · Related issues · -View source +View source @@ -1196,7 +1226,7 @@ NewAlias = TypeAliasType(get_name(), int) # error: TypeAliasType name mus Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1226,7 +1256,7 @@ TYPE_CHECKING = '' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1256,7 +1286,7 @@ b: Annotated[int] # `Annotated` expects at least two arguments Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -1290,7 +1320,7 @@ f(10) # Error Default level: error · Added in 0.0.1-alpha.11 · Related issues · -View source +View source @@ -1324,7 +1354,7 @@ class C: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1359,7 +1389,7 @@ T = TypeVar('T', bound=str) # valid bound TypeVar Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1384,7 +1414,7 @@ func() # TypeError: func() missing 1 required positional argument: 'x' Default level: error · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -1417,7 +1447,7 @@ alice["age"] # KeyError Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1446,7 +1476,7 @@ func("string") # error: [no-matching-overload] Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1470,7 +1500,7 @@ Subscripting an object that does not support it will raise a `TypeError` at runt Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1496,7 +1526,7 @@ for i in 34: # TypeError: 'int' object is not iterable Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1523,7 +1553,7 @@ f(1, x=2) # Error raised here Default level: error · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -1581,7 +1611,7 @@ def test(): -> "int": Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1611,7 +1641,7 @@ static_assert(int(2.0 * 3.0) == 6) # error: does not have a statically known tr Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1640,7 +1670,7 @@ class B(A): ... # Error raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1667,7 +1697,7 @@ f("foo") # Error raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1695,7 +1725,7 @@ def _(x: int): Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1741,7 +1771,7 @@ class A: Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1768,7 +1798,7 @@ f(x=1, y=2) # Error raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1796,7 +1826,7 @@ A().foo # AttributeError: 'A' object has no attribute 'foo' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1821,7 +1851,7 @@ import foo # ModuleNotFoundError: No module named 'foo' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1846,7 +1876,7 @@ print(x) # NameError: name 'x' is not defined Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1883,7 +1913,7 @@ b1 < b2 < b1 # exception raised here Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1911,7 +1941,7 @@ A() + A() # TypeError: unsupported operand type(s) for +: 'A' and 'A' Default level: error · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -1936,7 +1966,7 @@ l[1:10:0] # ValueError: slice step cannot be zero Default level: warn · Added in 0.0.1-alpha.20 · Related issues · -View source +View source @@ -1977,7 +2007,7 @@ class SubProto(BaseProto, Protocol): Default level: warn · Added in 0.0.1-alpha.16 · Related issues · -View source +View source @@ -2065,7 +2095,7 @@ a = 20 / 0 # type: ignore Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2093,7 +2123,7 @@ A.c # AttributeError: type object 'A' has no attribute 'c' Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2125,7 +2155,7 @@ A()[0] # TypeError: 'A' object is not subscriptable Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2157,7 +2187,7 @@ from module import a # ImportError: cannot import name 'a' from 'module' Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2184,7 +2214,7 @@ cast(int, f()) # Redundant Default level: warn · Added in 0.0.1-alpha.1 · Related issues · -View source +View source @@ -2208,7 +2238,7 @@ reveal_type(1) # NameError: name 'reveal_type' is not defined Default level: warn · Added in 0.0.1-alpha.15 · Related issues · -View source +View source @@ -2266,7 +2296,7 @@ def g(): Default level: warn · Added in 0.0.1-alpha.7 · Related issues · -View source +View source @@ -2305,7 +2335,7 @@ class D(C): ... # error: [unsupported-base] Default level: warn · Added in 0.0.1-alpha.22 · Related issues · -View source +View source @@ -2368,7 +2398,7 @@ def foo(x: int | str) -> int | str: Default level: ignore · Preview (since 0.0.1-alpha.1) · Related issues · -View source +View source @@ -2392,7 +2422,7 @@ Dividing by zero raises a `ZeroDivisionError` at runtime. Default level: ignore · Added in 0.0.1-alpha.1 · Related issues · -View source +View source diff --git a/crates/ty_ide/src/completion.rs b/crates/ty_ide/src/completion.rs index 4ea6cc18d98db..e2b03c393ed0c 100644 --- a/crates/ty_ide/src/completion.rs +++ b/crates/ty_ide/src/completion.rs @@ -127,7 +127,8 @@ impl<'db> Completion<'db> { Type::NominalInstance(_) | Type::PropertyInstance(_) | Type::BoundSuper(_) - | Type::TypedDict(_) => CompletionKind::Struct, + | Type::TypedDict(_) + | Type::NewTypeInstance(_) => CompletionKind::Struct, Type::IntLiteral(_) | Type::BooleanLiteral(_) | Type::TypeIs(_) diff --git a/crates/ty_ide/src/goto.rs b/crates/ty_ide/src/goto.rs index d7a7091f948d6..094d2008d2447 100644 --- a/crates/ty_ide/src/goto.rs +++ b/crates/ty_ide/src/goto.rs @@ -209,16 +209,11 @@ impl<'db> DefinitionsOrTargets<'db> { ty_python_semantic::types::TypeDefinition::Module(module) => { ResolvedDefinition::Module(module.file(db)?) } - ty_python_semantic::types::TypeDefinition::Class(definition) => { - ResolvedDefinition::Definition(definition) - } - ty_python_semantic::types::TypeDefinition::Function(definition) => { - ResolvedDefinition::Definition(definition) - } - ty_python_semantic::types::TypeDefinition::TypeVar(definition) => { - ResolvedDefinition::Definition(definition) - } - ty_python_semantic::types::TypeDefinition::TypeAlias(definition) => { + ty_python_semantic::types::TypeDefinition::Class(definition) + | ty_python_semantic::types::TypeDefinition::Function(definition) + | ty_python_semantic::types::TypeDefinition::TypeVar(definition) + | ty_python_semantic::types::TypeDefinition::TypeAlias(definition) + | ty_python_semantic::types::TypeDefinition::NewType(definition) => { ResolvedDefinition::Definition(definition) } }; diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/new_types.md b/crates/ty_python_semantic/resources/mdtest/annotations/new_types.md index 5dc14964ccb73..7a6e47ed32490 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/new_types.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/new_types.md @@ -1,7 +1,5 @@ # NewType -Currently, ty doesn't support `typing.NewType` in type annotations. - ## Valid forms ```py @@ -12,13 +10,389 @@ X = GenericAlias(type, ()) A = NewType("A", int) # TODO: typeshed for `typing.GenericAlias` uses `type` for the first argument. `NewType` should be special-cased # to be compatible with `type` -# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found `NewType`" +# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found ``" B = GenericAlias(A, ()) def _( a: A, b: B, ): - reveal_type(a) # revealed: @Todo(Support for `typing.NewType` instances in type expressions) + reveal_type(a) # revealed: A reveal_type(b) # revealed: @Todo(Support for `typing.GenericAlias` instances in type expressions) ``` + +## Subtyping + +The basic purpose of `NewType` is that it acts like a subtype of its base, but not the exact same +type (i.e. not an alias). + +```py +from typing_extensions import NewType +from ty_extensions import static_assert, is_subtype_of, is_equivalent_to + +Foo = NewType("Foo", int) +Bar = NewType("Bar", Foo) + +static_assert(is_subtype_of(Foo, int)) +static_assert(not is_equivalent_to(Foo, int)) + +static_assert(is_subtype_of(Bar, Foo)) +static_assert(is_subtype_of(Bar, int)) +static_assert(not is_equivalent_to(Bar, Foo)) + +Foo(42) +Foo(Foo(42)) # allowed: `Foo` is a subtype of `int`. +Foo(Bar(Foo(42))) # allowed: `Bar` is a subtype of `int`. +Foo(True) # allowed: `bool` is a subtype of `int`. +Foo("forty-two") # error: [invalid-argument-type] "Argument is incorrect: Expected `int`, found `Literal["forty-two"]`" + +def f(_: int): ... +def g(_: Foo): ... +def h(_: Bar): ... + +f(42) +f(Foo(42)) +f(Bar(Foo(42))) + +g(42) # error: [invalid-argument-type] "Argument to function `g` is incorrect: Expected `Foo`, found `Literal[42]`" +g(Foo(42)) +g(Bar(Foo(42))) + +h(42) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Literal[42]`" +h(Foo(42)) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Foo`" +h(Bar(Foo(42))) +``` + +## Member and method lookup work + +```py +from typing_extensions import NewType + +class Foo: + foo_member: str = "hello" + def foo_method(self) -> int: + return 42 + +Bar = NewType("Bar", Foo) +Baz = NewType("Baz", Bar) +baz = Baz(Bar(Foo())) +reveal_type(baz.foo_member) # revealed: str +reveal_type(baz.foo_method()) # revealed: int +``` + +We also infer member access on the `NewType` pseudo-type itself correctly: + +```py +reveal_type(Bar.__supertype__) # revealed: type | NewType +reveal_type(Baz.__supertype__) # revealed: type | NewType +``` + +## `NewType` wrapper functions are `Callable` + +```py +from collections.abc import Callable +from typing_extensions import NewType +from ty_extensions import CallableTypeOf + +Foo = NewType("Foo", int) + +def _(obj: CallableTypeOf[Foo]): + reveal_type(obj) # revealed: (int, /) -> Foo + +def f(_: Callable[[int], Foo]): ... + +f(Foo) +map(Foo, [1, 2, 3]) + +def g(_: Callable[[str], Foo]): ... + +g(Foo) # error: [invalid-argument-type] +``` + +## `NewType` instances are `Callable` if the base type is + +```py +from typing import NewType, Callable, Any +from ty_extensions import CallableTypeOf + +N = NewType("N", int) +i = N(42) + +y: Callable[..., Any] = i # error: [invalid-assignment] "Object of type `N` is not assignable to `(...) -> Any`" + +# error: [invalid-type-form] "Expected the first argument to `ty_extensions.CallableTypeOf` to be a callable object, but got an object of type `N`" +def f(x: CallableTypeOf[i]): + reveal_type(x) # revealed: Unknown + +class SomethingCallable: + def __call__(self, a: str) -> bytes: + raise NotImplementedError + +N2 = NewType("N2", SomethingCallable) +j = N2(SomethingCallable()) + +z: Callable[[str], bytes] = j # fine + +def g(x: CallableTypeOf[j]): + reveal_type(x) # revealed: (a: str) -> bytes +``` + +## The name must be a string literal + +```py +from typing_extensions import NewType + +def _(name: str) -> None: + _ = NewType(name, int) # error: [invalid-newtype] "The first argument to `NewType` must be a string literal" +``` + +However, the literal doesn't necessarily need to be inline, as long as we infer it: + +```py +name = "Foo" +Foo = NewType(name, int) +reveal_type(Foo) # revealed: +``` + +## The second argument must be a class type or another newtype + +Other typing constructs like `Union` are not allowed. + +```py +from typing_extensions import NewType + +# error: [invalid-newtype] "invalid base for `typing.NewType`" +Foo = NewType("Foo", int | str) +``` + +We don't emit the "invalid base" diagnostic for `Unknown`, because that typically results from other +errors that already have a diagnostic, and there's no need to pile on. For example, this mistake +gives you an "Int literals are not allowed" error, and we'd rather not see an "invalid base" error +on top of that: + +```py +# error: [invalid-type-form] "Int literals are not allowed in this context in a type expression" +Foo = NewType("Foo", 42) +``` + +## A `NewType` definition must be a simple variable assignment + +```py +from typing import NewType + +N: NewType = NewType("N", int) # error: [invalid-newtype] "A `NewType` definition must be a simple variable assignment" +``` + +## Newtypes can be cyclic in various ways + +Cyclic newtypes are kind of silly, but it's possible for the user to express them, and it's +important that we don't go into infinite recursive loops and crash with a stack overflow. In fact, +this is *why* base type evaluation is deferred; otherwise Salsa itself would crash. + +```py +from typing_extensions import NewType, reveal_type, cast + +# Define a directly cyclic newtype. +A = NewType("A", "A") +reveal_type(A) # revealed: + +# Typechecking still works. We can't construct an `A` "honestly", but we can `cast` into one. +a: A +a = 42 # error: [invalid-assignment] "Object of type `Literal[42]` is not assignable to `A`" +a = A(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `A`, found `Literal[42]`" +a = cast(A, 42) +reveal_type(a) # revealed: A + +# A newtype cycle might involve more than one step. +B = NewType("B", "C") +C = NewType("C", "B") +reveal_type(B) # revealed: +reveal_type(C) # revealed: +b: B = cast(B, 42) +c: C = C(b) +reveal_type(b) # revealed: B +reveal_type(c) # revealed: C +# Cyclic types behave in surprising ways. These assignments are legal, even though B and C aren't +# the same type, because each of them is a subtype of the other. +b = c +c = b + +# Another newtype could inherit from a cyclic one. +D = NewType("D", C) +reveal_type(D) # revealed: +d: D +d = D(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `C`, found `Literal[42]`" +d = D(c) +d = D(b) # Allowed, the same surprise as above. B and C are subtypes of each other. +reveal_type(d) # revealed: D +``` + +Normal classes can't inherit from newtypes, but generic classes can be parametrized with them, so we +also need to detect "ordinary" type cycles that happen to involve a newtype. + +```py +E = NewType("E", list["E"]) +reveal_type(E) # revealed: +e: E = E([]) +reveal_type(e) # revealed: E +reveal_type(E(E(E(E(E([])))))) # revealed: E +reveal_type(E([E([E([]), E([E([])])]), E([])])) # revealed: E +E(["foo"]) # error: [invalid-argument-type] +E(E(E(["foo"]))) # error: [invalid-argument-type] +``` + +## `NewType` wrapping preserves singleton-ness and single-valued-ness + +```py +from typing_extensions import NewType +from ty_extensions import is_singleton, is_single_valued, static_assert +from types import EllipsisType + +A = NewType("A", EllipsisType) +static_assert(is_singleton(A)) +static_assert(is_single_valued(A)) +reveal_type(type(A(...)) is EllipsisType) # revealed: Literal[True] +# TODO: This should be `Literal[True]` also. +reveal_type(A(...) is ...) # revealed: bool + +B = NewType("B", int) +static_assert(not is_singleton(B)) +static_assert(not is_single_valued(B)) +``` + +## `NewType`s of tuples can be iterated/unpacked + +```py +from typing import NewType + +N = NewType("N", tuple[int, str]) + +a, b = N((1, "foo")) + +reveal_type(a) # revealed: int +reveal_type(b) # revealed: str +``` + +## `isinstance` of a `NewType` instance and its base class is inferred as `Literal[True]` + +```py +from typing import NewType + +N = NewType("N", int) + +def f(x: N): + reveal_type(isinstance(x, int)) # revealed: Literal[True] +``` + +However, a `NewType` isn't a real class, so it isn't a valid second argument to `isinstance`: + +```py +def f(x: N): + # error: [invalid-argument-type] "Argument to function `isinstance` is incorrect" + reveal_type(isinstance(x, N)) # revealed: bool +``` + +Because of that, we don't generate any narrowing constraints for it: + +```py +def f(x: N | str): + if isinstance(x, N): # error: [invalid-argument-type] + reveal_type(x) # revealed: N | str + else: + reveal_type(x) # revealed: N | str +``` + +## Trying to subclass a `NewType` produces an error matching CPython + + + +```py +from typing import NewType + +X = NewType("X", int) + +class Foo(X): ... # error: [invalid-base] +``` + +## Don't narrow `NewType`-wrapped `Enum`s inside of match arms + +`Literal[Foo.X]` is actually disjoint from `N` here: + +```py +from enum import Enum +from typing import NewType + +class Foo(Enum): + X = 0 + Y = 1 + +N = NewType("N", Foo) + +def f(x: N): + match x: + case Foo.X: + reveal_type(x) # revealed: N + case Foo.Y: + reveal_type(x) # revealed: N + case _: + reveal_type(x) # revealed: N +``` + +## We don't support `NewType` on Python 3.9 + +We implement `typing.NewType` as a `KnownClass`, but in Python 3.9 it's actually a function, so all +we get is the `Any` annotations from typeshed. However, `typing_extensions.NewType` is always a +class. This could be improved in the future, but Python 3.9 is now end-of-life, so it's not +high-priority. + +```toml +[environment] +python-version = "3.9" +``` + +```py +from typing import NewType + +Foo = NewType("Foo", int) +reveal_type(Foo) # revealed: Any +reveal_type(Foo(42)) # revealed: Any + +from typing_extensions import NewType + +Bar = NewType("Bar", int) +reveal_type(Bar) # revealed: +reveal_type(Bar(42)) # revealed: Bar +``` + +## The base of a `NewType` can't be a protocol class or a `TypedDict` + + + +```py +from typing import NewType, Protocol, TypedDict + +class Id(Protocol): + code: int + +UserId = NewType("UserId", Id) # error: [invalid-newtype] + +class Foo(TypedDict): + a: int + +Bar = NewType("Bar", Foo) # error: [invalid-newtype] +``` + +## TODO: A `NewType` cannot be generic + +```py +from typing import Any, NewType, TypeVar + +# All of these are allowed. +A = NewType("A", list) +B = NewType("B", list[int]) +B = NewType("B", list[Any]) + +# But a free typevar is not allowed. +T = TypeVar("T") +C = NewType("C", list[T]) # TODO: should be "error: [invalid-newtype]" +``` diff --git a/crates/ty_python_semantic/resources/mdtest/class/super.md b/crates/ty_python_semantic/resources/mdtest/class/super.md index 5d4a4249b738d..80a4bc9806298 100644 --- a/crates/ty_python_semantic/resources/mdtest/class/super.md +++ b/crates/ty_python_semantic/resources/mdtest/class/super.md @@ -66,7 +66,7 @@ synthesized `Protocol`s that cannot be upcast to, or interpreted as, a non-`obje ```py import types -from typing_extensions import Callable, TypeIs, Literal, TypedDict +from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict def f(): ... @@ -81,6 +81,8 @@ class SomeTypedDict(TypedDict): x: int y: bytes +N = NewType("N", int) + # revealed: , FunctionType> reveal_type(super(object, f)) # revealed: , WrapperDescriptorType> @@ -95,6 +97,8 @@ reveal_type(super(object, Alias)) reveal_type(super(object, Foo().method)) # revealed: , property> reveal_type(super(object, Foo.some_property)) +# revealed: , int> +reveal_type(super(object, N(42))) def g(x: object) -> TypeIs[list[object]]: return isinstance(x, list) diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_The_base_of_a_`NewTy\342\200\246_(9847ea9eddc316b4).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_The_base_of_a_`NewTy\342\200\246_(9847ea9eddc316b4).snap" new file mode 100644 index 0000000000000..ee47accc9d333 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_The_base_of_a_`NewTy\342\200\246_(9847ea9eddc316b4).snap" @@ -0,0 +1,58 @@ +--- +source: crates/ty_test/src/lib.rs +expression: snapshot +--- +--- +mdtest name: new_types.md - NewType - The base of a `NewType` can't be a protocol class or a `TypedDict` +mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md +--- + +# Python source files + +## mdtest_snippet.py + +``` + 1 | from typing import NewType, Protocol, TypedDict + 2 | + 3 | class Id(Protocol): + 4 | code: int + 5 | + 6 | UserId = NewType("UserId", Id) # error: [invalid-newtype] + 7 | + 8 | class Foo(TypedDict): + 9 | a: int +10 | +11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype] +``` + +# Diagnostics + +``` +error[invalid-newtype]: invalid base for `typing.NewType` + --> src/mdtest_snippet.py:6:28 + | +4 | code: int +5 | +6 | UserId = NewType("UserId", Id) # error: [invalid-newtype] + | ^^ type `Id` +7 | +8 | class Foo(TypedDict): + | +info: The base of a `NewType` is not allowed to be a protocol class. +info: rule `invalid-newtype` is enabled by default + +``` + +``` +error[invalid-newtype]: invalid base for `typing.NewType` + --> src/mdtest_snippet.py:11:22 + | + 9 | a: int +10 | +11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype] + | ^^^ type `Foo` + | +info: The base of a `NewType` is not allowed to be a `TypedDict`. +info: rule `invalid-newtype` is enabled by default + +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_Trying_to_subclass_a\342\200\246_(fd3c73e2a9f04).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_Trying_to_subclass_a\342\200\246_(fd3c73e2a9f04).snap" new file mode 100644 index 0000000000000..9e4eac091a4fc --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/new_types.md_-_NewType_-_Trying_to_subclass_a\342\200\246_(fd3c73e2a9f04).snap" @@ -0,0 +1,37 @@ +--- +source: crates/ty_test/src/lib.rs +expression: snapshot +--- +--- +mdtest name: new_types.md - NewType - Trying to subclass a `NewType` produces an error matching CPython +mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md +--- + +# Python source files + +## mdtest_snippet.py + +``` +1 | from typing import NewType +2 | +3 | X = NewType("X", int) +4 | +5 | class Foo(X): ... # error: [invalid-base] +``` + +# Diagnostics + +``` +error[invalid-base]: Cannot subclass an instance of NewType + --> src/mdtest_snippet.py:5:11 + | +3 | X = NewType("X", int) +4 | +5 | class Foo(X): ... # error: [invalid-base] + | ^ + | +info: Perhaps you were looking for: `Foo = NewType('Foo', X)` +info: Definition of class `Foo` will raise `TypeError` at runtime +info: rule `invalid-base` is enabled by default + +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/super.md_-_Super_-_Basic_Usage_-_Explicit_Super_Objec\342\200\246_(b753048091f275c0).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/super.md_-_Super_-_Basic_Usage_-_Explicit_Super_Objec\342\200\246_(b753048091f275c0).snap" index 245c95d394f04..4a1c008f9fc19 100644 --- "a/crates/ty_python_semantic/resources/mdtest/snapshots/super.md_-_Super_-_Basic_Usage_-_Explicit_Super_Objec\342\200\246_(b753048091f275c0).snap" +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/super.md_-_Super_-_Basic_Usage_-_Explicit_Super_Objec\342\200\246_(b753048091f275c0).snap" @@ -46,7 +46,7 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md 32 | reveal_type(super(C, C()).aa) # revealed: int 33 | reveal_type(super(C, C()).bb) # revealed: int 34 | import types - 35 | from typing_extensions import Callable, TypeIs, Literal, TypedDict + 35 | from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict 36 | 37 | def f(): ... 38 | @@ -61,59 +61,63 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md 47 | x: int 48 | y: bytes 49 | - 50 | # revealed: , FunctionType> - 51 | reveal_type(super(object, f)) - 52 | # revealed: , WrapperDescriptorType> - 53 | reveal_type(super(object, types.FunctionType.__get__)) - 54 | # revealed: , GenericAlias> - 55 | reveal_type(super(object, Foo[int])) - 56 | # revealed: , _SpecialForm> - 57 | reveal_type(super(object, Literal)) - 58 | # revealed: , TypeAliasType> - 59 | reveal_type(super(object, Alias)) - 60 | # revealed: , MethodType> - 61 | reveal_type(super(object, Foo().method)) - 62 | # revealed: , property> - 63 | reveal_type(super(object, Foo.some_property)) - 64 | - 65 | def g(x: object) -> TypeIs[list[object]]: - 66 | return isinstance(x, list) - 67 | - 68 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]): - 69 | if hasattr(x, "bar"): - 70 | # revealed: - 71 | reveal_type(x) - 72 | # error: [invalid-super-argument] - 73 | # revealed: Unknown - 74 | reveal_type(super(object, x)) - 75 | - 76 | # error: [invalid-super-argument] - 77 | # revealed: Unknown - 78 | reveal_type(super(object, z)) + 50 | N = NewType("N", int) + 51 | + 52 | # revealed: , FunctionType> + 53 | reveal_type(super(object, f)) + 54 | # revealed: , WrapperDescriptorType> + 55 | reveal_type(super(object, types.FunctionType.__get__)) + 56 | # revealed: , GenericAlias> + 57 | reveal_type(super(object, Foo[int])) + 58 | # revealed: , _SpecialForm> + 59 | reveal_type(super(object, Literal)) + 60 | # revealed: , TypeAliasType> + 61 | reveal_type(super(object, Alias)) + 62 | # revealed: , MethodType> + 63 | reveal_type(super(object, Foo().method)) + 64 | # revealed: , property> + 65 | reveal_type(super(object, Foo.some_property)) + 66 | # revealed: , int> + 67 | reveal_type(super(object, N(42))) + 68 | + 69 | def g(x: object) -> TypeIs[list[object]]: + 70 | return isinstance(x, list) + 71 | + 72 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]): + 73 | if hasattr(x, "bar"): + 74 | # revealed: + 75 | reveal_type(x) + 76 | # error: [invalid-super-argument] + 77 | # revealed: Unknown + 78 | reveal_type(super(object, x)) 79 | - 80 | is_list = g(x) - 81 | # revealed: TypeIs[list[object] @ x] - 82 | reveal_type(is_list) - 83 | # revealed: , bool> - 84 | reveal_type(super(object, is_list)) - 85 | - 86 | # revealed: , dict[Literal["x", "y"], int | bytes]> - 87 | reveal_type(super(object, y)) - 88 | - 89 | # The first argument to `super()` must be an actual class object; - 90 | # instances of `GenericAlias` are not accepted at runtime: - 91 | # - 92 | # error: [invalid-super-argument] - 93 | # revealed: Unknown - 94 | reveal_type(super(list[int], [])) - 95 | class Super: - 96 | def method(self) -> int: - 97 | return 42 - 98 | - 99 | class Sub(Super): -100 | def method(self: Sub) -> int: -101 | # revealed: , Sub> -102 | return reveal_type(super(self.__class__, self)).method() + 80 | # error: [invalid-super-argument] + 81 | # revealed: Unknown + 82 | reveal_type(super(object, z)) + 83 | + 84 | is_list = g(x) + 85 | # revealed: TypeIs[list[object] @ x] + 86 | reveal_type(is_list) + 87 | # revealed: , bool> + 88 | reveal_type(super(object, is_list)) + 89 | + 90 | # revealed: , dict[Literal["x", "y"], int | bytes]> + 91 | reveal_type(super(object, y)) + 92 | + 93 | # The first argument to `super()` must be an actual class object; + 94 | # instances of `GenericAlias` are not accepted at runtime: + 95 | # + 96 | # error: [invalid-super-argument] + 97 | # revealed: Unknown + 98 | reveal_type(super(list[int], [])) + 99 | class Super: +100 | def method(self) -> int: +101 | return 42 +102 | +103 | class Sub(Super): +104 | def method(self: Sub) -> int: +105 | # revealed: , Sub> +106 | return reveal_type(super(self.__class__, self)).method() ``` # Diagnostics @@ -206,14 +210,14 @@ info: rule `unresolved-attribute` is enabled by default ``` error[invalid-super-argument]: `` is an abstract/structural type in `super(, )` call - --> src/mdtest_snippet.py:74:21 + --> src/mdtest_snippet.py:78:21 | -72 | # error: [invalid-super-argument] -73 | # revealed: Unknown -74 | reveal_type(super(object, x)) +76 | # error: [invalid-super-argument] +77 | # revealed: Unknown +78 | reveal_type(super(object, x)) | ^^^^^^^^^^^^^^^^ -75 | -76 | # error: [invalid-super-argument] +79 | +80 | # error: [invalid-super-argument] | info: rule `invalid-super-argument` is enabled by default @@ -221,14 +225,14 @@ info: rule `invalid-super-argument` is enabled by default ``` error[invalid-super-argument]: `(int, str, /) -> bool` is an abstract/structural type in `super(, (int, str, /) -> bool)` call - --> src/mdtest_snippet.py:78:17 + --> src/mdtest_snippet.py:82:17 | -76 | # error: [invalid-super-argument] -77 | # revealed: Unknown -78 | reveal_type(super(object, z)) +80 | # error: [invalid-super-argument] +81 | # revealed: Unknown +82 | reveal_type(super(object, z)) | ^^^^^^^^^^^^^^^^ -79 | -80 | is_list = g(x) +83 | +84 | is_list = g(x) | info: rule `invalid-super-argument` is enabled by default @@ -236,15 +240,15 @@ info: rule `invalid-super-argument` is enabled by default ``` error[invalid-super-argument]: `types.GenericAlias` instance `list[int]` is not a valid class - --> src/mdtest_snippet.py:94:13 - | -92 | # error: [invalid-super-argument] -93 | # revealed: Unknown -94 | reveal_type(super(list[int], [])) - | ^^^^^^^^^^^^^^^^^^^^ -95 | class Super: -96 | def method(self) -> int: - | + --> src/mdtest_snippet.py:98:13 + | + 96 | # error: [invalid-super-argument] + 97 | # revealed: Unknown + 98 | reveal_type(super(list[int], [])) + | ^^^^^^^^^^^^^^^^^^^^ + 99 | class Super: +100 | def method(self) -> int: + | info: rule `invalid-super-argument` is enabled by default ``` diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index c9641c4e34695..0d0879f348869 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -66,6 +66,7 @@ use crate::types::generics::{ use crate::types::infer::infer_unpack_types; use crate::types::mro::{Mro, MroError, MroIterator}; pub(crate) use crate::types::narrow::infer_narrowing_constraint; +use crate::types::newtype::NewType; use crate::types::signatures::{ParameterForm, walk_signature}; use crate::types::tuple::{TupleSpec, TupleSpecBuilder}; pub(crate) use crate::types::typed_dict::{TypedDictParams, TypedDictType, walk_typed_dict_type}; @@ -98,6 +99,7 @@ mod instance; mod member; mod mro; mod narrow; +mod newtype; mod protocol_class; mod signatures; mod special_form; @@ -783,6 +785,13 @@ pub enum Type<'db> { TypedDict(TypedDictType<'db>), /// An aliased type (lazily not-yet-unpacked to its value type). TypeAlias(TypeAliasType<'db>), + /// The set of Python objects that belong to a `typing.NewType` subtype. Note that + /// `typing.NewType` itself is a `Type::ClassLiteral` with `KnownClass::NewType`, and the + /// identity callables it returns (which behave like subtypes in type expressions) are of + /// `Type::KnownInstance` with `KnownInstanceType::NewType`. This `Type` refers to the objects + /// wrapped/returned by a specific one of those identity callables, or by another that inherits + /// from it. + NewTypeInstance(NewType<'db>), } #[salsa::tracked] @@ -1420,6 +1429,13 @@ impl<'db> Type<'db> { self } Type::TypeAlias(alias) => alias.value_type(db).normalized_impl(db, visitor), + Type::NewTypeInstance(newtype) => { + visitor.visit(self, || { + Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| { + class_type.normalized_impl(db, visitor) + })) + }) + } Type::LiteralString | Type::AlwaysFalsy | Type::AlwaysTruthy @@ -1482,7 +1498,8 @@ impl<'db> Type<'db> { | Type::BoundSuper(_) | Type::TypeIs(_) | Type::TypedDict(_) - | Type::TypeAlias(_) => false, + | Type::TypeAlias(_) + | Type::NewTypeInstance(_) => false, } } @@ -1520,6 +1537,10 @@ impl<'db> Type<'db> { Type::GenericAlias(alias) => Some(ClassType::Generic(alias).into_callable(db)), + Type::NewTypeInstance(newtype) => { + Type::instance(db, newtype.base_class_type(db)).try_upcast_to_callable(db) + } + // TODO: This is unsound so in future we can consider an opt-in option to disable it. Type::SubclassOf(subclass_of_ty) => match subclass_of_ty.subclass_of() { SubclassOfInner::Class(class) => Some(class.into_callable(db)), @@ -1549,6 +1570,15 @@ impl<'db> Type<'db> { false, ))), + Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Some(CallableType::single( + db, + Signature::new( + Parameters::new([Parameter::positional_only(None) + .with_annotated_type(newtype.base(db).instance_type(db))]), + Some(Type::NewTypeInstance(newtype)), + ), + )), + Type::Never | Type::DataclassTransformer(_) | Type::AlwaysTruthy @@ -2429,6 +2459,22 @@ impl<'db> Type<'db> { }) } + (Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(target_newtype)) => { + self_newtype.has_relation_to_impl(db, target_newtype) + } + + ( + Type::NewTypeInstance(self_newtype), + Type::NominalInstance(target_nominal_instance), + ) => self_newtype.base_class_type(db).has_relation_to_impl( + db, + target_nominal_instance.class(db), + inferable, + relation, + relation_visitor, + disjointness_visitor, + ), + (Type::PropertyInstance(_), _) => { KnownClass::Property.to_instance(db).has_relation_to_impl( db, @@ -2448,14 +2494,15 @@ impl<'db> Type<'db> { disjointness_visitor, ), - // Other than the special cases enumerated above, `Instance` types and typevars are - // never subtypes of any other variants + // Other than the special cases enumerated above, nominal-instance types, + // newtype-instance types, and typevars are never subtypes of any other variants (Type::TypeVar(bound_typevar), _) => { // All inferable cases should have been handled above assert!(!bound_typevar.is_inferable(db, inferable)); ConstraintSet::from(false) } (Type::NominalInstance(_), _) => ConstraintSet::from(false), + (Type::NewTypeInstance(_), _) => ConstraintSet::from(false), } } @@ -2529,6 +2576,10 @@ impl<'db> Type<'db> { }) } + (Type::NewTypeInstance(self_newtype), Type::NewTypeInstance(other_newtype)) => { + ConstraintSet::from(self_newtype.is_equivalent_to_impl(db, other_newtype)) + } + (Type::NominalInstance(first), Type::NominalInstance(second)) => { first.is_equivalent_to_impl(db, second, inferable, visitor) } @@ -3288,6 +3339,19 @@ impl<'db> Type<'db> { ) }), + (Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => { + left.is_disjoint_from_impl(db, right) + } + (Type::NewTypeInstance(newtype), other) | (other, Type::NewTypeInstance(newtype)) => { + Type::instance(db, newtype.base_class_type(db)).is_disjoint_from_impl( + db, + other, + inferable, + disjointness_visitor, + relation_visitor, + ) + } + (Type::PropertyInstance(_), other) | (other, Type::PropertyInstance(_)) => { KnownClass::Property.to_instance(db).is_disjoint_from_impl( db, @@ -3432,6 +3496,9 @@ impl<'db> Type<'db> { Type::TypeIs(type_is) => type_is.is_bound(db), Type::TypedDict(_) => false, Type::TypeAlias(alias) => alias.value_type(db).is_singleton(db), + Type::NewTypeInstance(newtype) => { + Type::instance(db, newtype.base_class_type(db)).is_singleton(db) + } } } @@ -3482,6 +3549,9 @@ impl<'db> Type<'db> { } Type::NominalInstance(instance) => instance.is_single_valued(db), + Type::NewTypeInstance(newtype) => { + Type::instance(db, newtype.base_class_type(db)).is_single_valued(db) + } Type::BoundSuper(_) => { // At runtime two super instances never compare equal, even if their arguments are identical. @@ -3645,7 +3715,8 @@ impl<'db> Type<'db> { | Type::ProtocolInstance(_) | Type::PropertyInstance(_) | Type::TypeIs(_) - | Type::TypedDict(_) => None, + | Type::TypedDict(_) + | Type::NewTypeInstance(_) => None, } } @@ -3732,6 +3803,7 @@ impl<'db> Type<'db> { Type::Dynamic(_) | Type::Never => Place::bound(self).into(), Type::NominalInstance(instance) => instance.class(db).instance_member(db, name), + Type::NewTypeInstance(newtype) => newtype.base_class_type(db).instance_member(db, name), Type::ProtocolInstance(protocol) => protocol.instance_member(db, name), @@ -4404,6 +4476,7 @@ impl<'db> Type<'db> { Type::NominalInstance(..) | Type::ProtocolInstance(..) + | Type::NewTypeInstance(..) | Type::BooleanLiteral(..) | Type::IntLiteral(..) | Type::StringLiteral(..) @@ -4842,6 +4915,8 @@ impl<'db> Type<'db> { .value_type(db) .try_bool_impl(db, allow_short_circuit, visitor) })?, + Type::NewTypeInstance(newtype) => Type::instance(db, newtype.base_class_type(db)) + .try_bool_impl(db, allow_short_circuit, visitor)?, }; Ok(truthiness) @@ -5528,7 +5603,7 @@ impl<'db> Type<'db> { SubclassOfInner::Class(class) => Type::from(class).bindings(db), }, - Type::NominalInstance(_) | Type::ProtocolInstance(_) => { + Type::NominalInstance(_) | Type::ProtocolInstance(_) | Type::NewTypeInstance(_) => { // Note that for objects that have a (possibly not callable!) `__call__` attribute, // we will get the signature of the `__call__` attribute, but will pass in the type // of the original object as the "callable type". That ensures that we get errors @@ -5581,6 +5656,16 @@ impl<'db> Type<'db> { Type::EnumLiteral(enum_literal) => enum_literal.enum_class_instance(db).bindings(db), + Type::KnownInstance(KnownInstanceType::NewType(newtype)) => Binding::single( + self, + Signature::new( + Parameters::new([Parameter::positional_only(None) + .with_annotated_type(newtype.base(db).instance_type(db))]), + Some(Type::NewTypeInstance(newtype)), + ), + ) + .into(), + Type::KnownInstance(known_instance) => { known_instance.instance_fallback(db).bindings(db) } @@ -5716,6 +5801,7 @@ impl<'db> Type<'db> { match ty { Type::NominalInstance(nominal) => nominal.tuple_spec(db), + Type::NewTypeInstance(newtype) => non_async_special_case(db, Type::instance(db, newtype.base_class_type(db))), Type::GenericAlias(alias) if alias.origin(db).is_tuple(db) => { Some(Cow::Owned(TupleSpec::homogeneous(todo_type!( "*tuple[] annotations" @@ -6346,6 +6432,9 @@ impl<'db> Type<'db> { Type::ClassLiteral(class) => Some(Type::instance(db, class.default_specialization(db))), Type::GenericAlias(alias) => Some(Type::instance(db, ClassType::from(alias))), Type::SubclassOf(subclass_of_ty) => Some(subclass_of_ty.to_instance(db)), + Type::KnownInstance(KnownInstanceType::NewType(newtype)) => { + Some(Type::NewTypeInstance(newtype)) + } Type::Union(union) => union.to_instance(db), // If there is no bound or constraints on a typevar `T`, `T: object` implicitly, which // has no instance type. Otherwise, synthesize a typevar with bound or constraints @@ -6376,7 +6465,8 @@ impl<'db> Type<'db> { | Type::AlwaysTruthy | Type::AlwaysFalsy | Type::TypeIs(_) - | Type::TypedDict(_) => None, + | Type::TypedDict(_) + | Type::NewTypeInstance(_) => None, } } @@ -6455,6 +6545,7 @@ impl<'db> Type<'db> { Type::KnownInstance(known_instance) => match known_instance { KnownInstanceType::TypeAliasType(alias) => Ok(Type::TypeAlias(*alias)), + KnownInstanceType::NewType(newtype) => Ok(Type::NewTypeInstance(*newtype)), KnownInstanceType::TypeVar(typevar) => { let index = semantic_index(db, scope_id.file(db)); Ok(bind_typevar( @@ -6669,9 +6760,6 @@ impl<'db> Type<'db> { Some(KnownClass::TypeVarTuple) => Ok(todo_type!( "Support for `typing.TypeVarTuple` instances in type expressions" )), - Some(KnownClass::NewType) => Ok(todo_type!( - "Support for `typing.NewType` instances in type expressions" - )), Some(KnownClass::GenericAlias) => Ok(todo_type!( "Support for `typing.GenericAlias` instances in type expressions" )), @@ -6690,6 +6778,13 @@ impl<'db> Type<'db> { .value_type(db) .in_type_expression(db, scope_id, typevar_binding_context) } + + Type::NewTypeInstance(_) => Err(InvalidTypeExpressionError { + invalid_expressions: smallvec::smallvec_inline![ + InvalidTypeExpression::InvalidType(*self, scope_id) + ], + fallback_type: Type::unknown(), + }), } } @@ -6764,6 +6859,7 @@ impl<'db> Type<'db> { // understand a more specific meta type in order to correctly handle `__getitem__`. Type::TypedDict(typed_dict) => SubclassOfType::from(db, typed_dict.defining_class()), Type::TypeAlias(alias) => alias.value_type(db).to_meta_type(db), + Type::NewTypeInstance(newtype) => Type::from(newtype.base_class_type(db)), } } @@ -6873,8 +6969,8 @@ impl<'db> Type<'db> { | TypeMapping::ReplaceParameterDefaults | TypeMapping::BindLegacyTypevars(_) => self, TypeMapping::Materialize(materialization_kind) => { - Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor)) - } + Type::TypeVar(bound_typevar.materialize_impl(db, *materialization_kind, visitor)) + } } Type::KnownInstance(KnownInstanceType::TypeVar(typevar)) => match type_mapping { @@ -6909,6 +7005,12 @@ impl<'db> Type<'db> { instance.apply_type_mapping_impl(db, type_mapping, tcx, visitor) }, + Type::NewTypeInstance(newtype) => visitor.visit(self, || { + Type::NewTypeInstance(newtype.map_base_class_type(db, |class_type| { + class_type.apply_type_mapping_impl(db, type_mapping, tcx, visitor) + })) + }), + Type::ProtocolInstance(instance) => { // TODO: Add tests for materialization once subtyping/assignability is implemented for // protocols. It _might_ require changing the logic here because: @@ -7150,6 +7252,12 @@ impl<'db> Type<'db> { instance.find_legacy_typevars_impl(db, binding_context, typevars, visitor); } + Type::NewTypeInstance(_) => { + // A newtype can never be constructed from an unspecialized generic class, so it is + // impossible that we could ever find any legacy typevars in a newtype instance or + // its underlying class. + } + Type::SubclassOf(subclass_of) => { subclass_of.find_legacy_typevars_impl(db, binding_context, typevars, visitor); } @@ -7305,6 +7413,7 @@ impl<'db> Type<'db> { }, Self::TypeAlias(alias) => alias.value_type(db).definition(db), + Self::NewTypeInstance(newtype) => Some(TypeDefinition::NewType(newtype.definition(db))), Self::StringLiteral(_) | Self::BooleanLiteral(_) @@ -7528,7 +7637,8 @@ impl<'db> VarianceInferable<'db> for Type<'db> { | Type::BoundSuper(_) | Type::TypeVar(_) | Type::TypedDict(_) - | Type::TypeAlias(_) => TypeVarVariance::Bivariant, + | Type::TypeAlias(_) + | Type::NewTypeInstance(_) => TypeVarVariance::Bivariant, }; tracing::trace!( @@ -7726,6 +7836,10 @@ pub enum KnownInstanceType<'db> { /// A single instance of `typing.Annotated` Annotated(InternedType<'db>), + + /// An identity callable created with `typing.NewType(name, base)`, which behaves like a + /// subtype of `base` in type expressions. See the `struct NewType` payload for an example. + NewType(NewType<'db>), } fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( @@ -7760,6 +7874,11 @@ fn walk_known_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( KnownInstanceType::Literal(ty) | KnownInstanceType::Annotated(ty) => { visitor.visit_type(db, ty.inner(db)); } + KnownInstanceType::NewType(newtype) => { + if let ClassType::Generic(generic_alias) = newtype.base_class_type(db) { + visitor.visit_generic_alias_type(db, generic_alias); + } + } } } @@ -7799,6 +7918,10 @@ impl<'db> KnownInstanceType<'db> { Self::UnionType(list) => Self::UnionType(list.normalized_impl(db, visitor)), Self::Literal(ty) => Self::Literal(ty.normalized_impl(db, visitor)), Self::Annotated(ty) => Self::Annotated(ty.normalized_impl(db, visitor)), + Self::NewType(newtype) => Self::NewType( + newtype + .map_base_class_type(db, |class_type| class_type.normalized_impl(db, visitor)), + ), } } @@ -7819,6 +7942,7 @@ impl<'db> KnownInstanceType<'db> { Self::UnionType(_) => KnownClass::UnionType, Self::Literal(_) => KnownClass::GenericAlias, Self::Annotated(_) => KnownClass::GenericAlias, + Self::NewType(_) => KnownClass::NewType, } } @@ -7903,6 +8027,9 @@ impl<'db> KnownInstanceType<'db> { KnownInstanceType::Annotated(_) => { f.write_str("") } + KnownInstanceType::NewType(declaration) => { + write!(f, "", declaration.name(self.db)) + } } } } diff --git a/crates/ty_python_semantic/src/types/bound_super.rs b/crates/ty_python_semantic/src/types/bound_super.rs index 011318db51e22..24d6573c28c4c 100644 --- a/crates/ty_python_semantic/src/types/bound_super.rs +++ b/crates/ty_python_semantic/src/types/bound_super.rs @@ -404,6 +404,9 @@ impl<'db> BoundSuperType<'db> { .to_specialized_instance(db, [key_builder.build(), value_builder.build()]), ); } + Type::NewTypeInstance(newtype) => { + return delegate_to(Type::instance(db, newtype.base_class_type(db))); + } Type::Callable(callable) if callable.is_function_like(db) => { return delegate_to(KnownClass::FunctionType.to_instance(db)); } diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 18d3d4b900eac..ce6fe0d19bdbb 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -358,6 +358,14 @@ pub enum ClassType<'db> { #[salsa::tracked] impl<'db> ClassType<'db> { + /// Return a `ClassType` representing the class `builtins.object` + pub(super) fn object(db: &'db dyn Db) -> Self { + KnownClass::Object + .to_class_literal(db) + .to_class_type(db) + .unwrap() + } + pub(super) const fn is_generic(self) -> bool { matches!(self, Self::Generic(_)) } diff --git a/crates/ty_python_semantic/src/types/class_base.rs b/crates/ty_python_semantic/src/types/class_base.rs index 87417f83144df..9cc09acc0f4f6 100644 --- a/crates/ty_python_semantic/src/types/class_base.rs +++ b/crates/ty_python_semantic/src/types/class_base.rs @@ -137,6 +137,12 @@ impl<'db> ClassBase<'db> { Type::TypeAlias(alias) => Self::try_from_type(db, alias.value_type(db), subclass), + Type::NewTypeInstance(newtype) => ClassBase::try_from_type( + db, + Type::instance(db, newtype.base_class_type(db)), + subclass, + ), + Type::PropertyInstance(_) | Type::BooleanLiteral(_) | Type::FunctionLiteral(_) @@ -169,7 +175,11 @@ impl<'db> ClassBase<'db> { | KnownInstanceType::Field(_) | KnownInstanceType::ConstraintSet(_) | KnownInstanceType::UnionType(_) - | KnownInstanceType::Literal(_) => None, + | KnownInstanceType::Literal(_) + // A class inheriting from a newtype would make intuitive sense, but newtype + // wrappers are just identity callables at runtime, so this sort of inheritance + // doesn't work and isn't allowed. + | KnownInstanceType::NewType(_) => None, KnownInstanceType::Annotated(ty) => Self::try_from_type(db, ty.inner(db), subclass), }, diff --git a/crates/ty_python_semantic/src/types/definition.rs b/crates/ty_python_semantic/src/types/definition.rs index f98d47ba93777..9095dcea442fb 100644 --- a/crates/ty_python_semantic/src/types/definition.rs +++ b/crates/ty_python_semantic/src/types/definition.rs @@ -12,6 +12,7 @@ pub enum TypeDefinition<'db> { Function(Definition<'db>), TypeVar(Definition<'db>), TypeAlias(Definition<'db>), + NewType(Definition<'db>), } impl TypeDefinition<'_> { @@ -21,7 +22,8 @@ impl TypeDefinition<'_> { Self::Class(definition) | Self::Function(definition) | Self::TypeVar(definition) - | Self::TypeAlias(definition) => { + | Self::TypeAlias(definition) + | Self::NewType(definition) => { let module = parsed_module(db, definition.file(db)).load(db); Some(definition.focus_range(db, &module)) } @@ -38,7 +40,8 @@ impl TypeDefinition<'_> { Self::Class(definition) | Self::Function(definition) | Self::TypeVar(definition) - | Self::TypeAlias(definition) => { + | Self::TypeAlias(definition) + | Self::NewType(definition) => { let module = parsed_module(db, definition.file(db)).load(db); Some(definition.full_range(db, &module)) } diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 33efcc74fd526..ccb0c824723e3 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -12,6 +12,7 @@ use crate::semantic_index::definition::{Definition, DefinitionKind}; use crate::semantic_index::place::{PlaceTable, ScopedPlaceId}; use crate::semantic_index::{global_scope, place_table}; use crate::suppression::FileSuppressionId; +use crate::types::KnownInstanceType; use crate::types::call::CallError; use crate::types::class::{DisjointBase, DisjointBaseKind, Field}; use crate::types::function::KnownFunction; @@ -65,6 +66,7 @@ pub(crate) fn register_lints(registry: &mut LintRegistryBuilder) { registry.register_lint(&INVALID_LEGACY_TYPE_VARIABLE); registry.register_lint(&INVALID_PARAMSPEC); registry.register_lint(&INVALID_TYPE_ALIAS_TYPE); + registry.register_lint(&INVALID_NEWTYPE); registry.register_lint(&INVALID_METACLASS); registry.register_lint(&INVALID_OVERLOAD); registry.register_lint(&USELESS_OVERLOAD_BODY); @@ -926,6 +928,30 @@ declare_lint! { } } +declare_lint! { + /// ## What it does + /// Checks for the creation of invalid `NewType`s + /// + /// ## Why is this bad? + /// There are several requirements that you must follow when creating a `NewType`. + /// + /// ## Examples + /// ```python + /// from typing import NewType + /// + /// def get_name() -> str: ... + /// + /// Foo = NewType("Foo", int) # okay + /// Bar = NewType(get_name(), int) # error: The first argument to `NewType` must be a string literal + /// Baz = NewType("Baz", int | str) # error: invalid base for `typing.NewType` + /// ``` + pub(crate) static INVALID_NEWTYPE = { + summary: "detects invalid NewType definitions", + status: LintStatus::preview("1.0.0"), + default_level: Level::Error, + } +} + declare_lint! { /// ## What it does /// Checks for arguments to `metaclass=` that are invalid. @@ -2898,6 +2924,24 @@ pub(crate) fn report_invalid_or_unsupported_base( return; } + if let Type::KnownInstance(KnownInstanceType::NewType(newtype)) = base_type { + let Some(builder) = context.report_lint(&INVALID_BASE, base_node) else { + return; + }; + let mut diagnostic = builder.into_diagnostic("Cannot subclass an instance of NewType"); + diagnostic.info(format_args!( + "Perhaps you were looking for: `{} = NewType('{}', {})`", + class.name(context.db()), + class.name(context.db()), + newtype.name(context.db()), + )); + diagnostic.info(format_args!( + "Definition of class `{}` will raise `TypeError` at runtime", + class.name(context.db()) + )); + return; + } + let tuple_of_types = Type::homogeneous_tuple(db, instance_of_type); let explain_mro_entries = |diagnostic: &mut LintDiagnosticGuard| { diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 42e237313440f..b8a8a05ac4ca0 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -618,6 +618,7 @@ impl Display for DisplayRepresentation<'_> { .fmt(f), } } + Type::NewTypeInstance(newtype) => f.write_str(newtype.name(self.db)), } } } diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index 2462748d03f0c..98a86f48df7a5 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1101,6 +1101,11 @@ fn is_instance_truthiness<'db>( Type::NominalInstance(..) => always_true_if(is_instance(&ty)), + Type::NewTypeInstance(newtype) => always_true_if(is_instance(&Type::instance( + db, + newtype.base_class_type(db), + ))), + Type::BooleanLiteral(..) | Type::BytesLiteral(..) | Type::IntLiteral(..) diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 94cb15993c2bd..cd29efaeacb94 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -128,6 +128,10 @@ impl<'db> AllMembers<'db> { } } + Type::NewTypeInstance(newtype) => { + self.extend_with_type(db, Type::instance(db, newtype.base_class_type(db))); + } + Type::ClassLiteral(class_literal) if class_literal.is_typed_dict(db) => { self.extend_with_type(db, KnownClass::TypedDictFallback.to_class_literal(db)); } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index c2274127524d4..2cb4bf327411d 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -59,8 +59,8 @@ use crate::types::diagnostic::{ DIVISION_BY_ZERO, DUPLICATE_KW_ONLY, INCONSISTENT_MRO, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_BASE, INVALID_DECLARATION, INVALID_GENERIC_CLASS, INVALID_KEY, INVALID_LEGACY_TYPE_VARIABLE, INVALID_METACLASS, - INVALID_NAMED_TUPLE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT, INVALID_PARAMSPEC, - INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL, + INVALID_NAMED_TUPLE, INVALID_NEWTYPE, INVALID_OVERLOAD, INVALID_PARAMETER_DEFAULT, + INVALID_PARAMSPEC, INVALID_PROTOCOL, INVALID_TYPE_FORM, INVALID_TYPE_GUARD_CALL, INVALID_TYPE_VARIABLE_CONSTRAINTS, IncompatibleBases, NON_SUBSCRIPTABLE, POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_IMPORT, SUBCLASS_OF_FINAL_CLASS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_IMPORT, @@ -90,6 +90,7 @@ use crate::types::generics::{ use crate::types::infer::nearest_enclosing_function; use crate::types::instance::SliceLiteral; use crate::types::mro::MroErrorKind; +use crate::types::newtype::NewType; use crate::types::signatures::Signature; use crate::types::subclass_of::SubclassOfInner; use crate::types::tuple::{Tuple, TupleLength, TupleSpec, TupleType}; @@ -3884,7 +3885,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | Type::AlwaysTruthy | Type::AlwaysFalsy | Type::TypeIs(_) - | Type::TypedDict(_) => { + | Type::TypedDict(_) + | Type::NewTypeInstance(_) => { // TODO: We could use the annotated parameter type of `__setattr__` as type context here. // However, we would still have to perform the first inference without type context. let value_ty = infer_value_ty(self, TypeContext::default()); @@ -4454,6 +4456,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Some(KnownClass::ParamSpec) => { self.infer_paramspec(target, call_expr, definition) } + Some(KnownClass::NewType) => { + self.infer_newtype_expression(target, call_expr, definition) + } Some(_) | None => { self.infer_call_expression_impl(call_expr, callable_type, tcx) } @@ -4892,14 +4897,114 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ))) } + fn infer_newtype_expression( + &mut self, + target: &ast::Expr, + call_expr: &ast::ExprCall, + definition: Definition<'db>, + ) -> Type<'db> { + fn error<'db>( + context: &InferContext<'db, '_>, + message: impl std::fmt::Display, + node: impl Ranged, + ) -> Type<'db> { + if let Some(builder) = context.report_lint(&INVALID_NEWTYPE, node) { + builder.into_diagnostic(message); + } + Type::unknown() + } + + let db = self.db(); + let arguments = &call_expr.arguments; + + if !arguments.keywords.is_empty() { + return error( + &self.context, + "Keyword arguments are not supported in `NewType` creation", + call_expr, + ); + } + + if let Some(starred) = arguments.args.iter().find(|arg| arg.is_starred_expr()) { + return error( + &self.context, + "Starred arguments are not supported in `NewType` creation", + starred, + ); + } + + if arguments.args.len() != 2 { + return error( + &self.context, + format!( + "Wrong number of arguments in `NewType` creation, expected 2, found {}", + arguments.args.len() + ), + call_expr, + ); + } + + let name_param_ty = self.infer_expression(&arguments.args[0], TypeContext::default()); + + let Some(name) = name_param_ty.as_string_literal().map(|name| name.value(db)) else { + return error( + &self.context, + "The first argument to `NewType` must be a string literal", + call_expr, + ); + }; + + let ast::Expr::Name(ast::ExprName { + id: target_name, .. + }) = target + else { + return error( + &self.context, + "A `NewType` definition must be a simple variable assignment", + target, + ); + }; + + if name != target_name { + return error( + &self.context, + format_args!( + "The name of a `NewType` (`{name}`) must match \ + the name of the variable it is assigned to (`{target_name}`)" + ), + target, + ); + } + + // Inference of `tp` must be deferred, to avoid cycles. + self.deferred.insert(definition, self.multi_inference_state); + + Type::KnownInstance(KnownInstanceType::NewType(NewType::new( + db, + ast::name::Name::from(name), + definition, + None, + ))) + } + fn infer_assignment_deferred(&mut self, value: &ast::Expr) { - // Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec. + // Infer deferred bounds/constraints/defaults of a legacy TypeVar / ParamSpec / NewType. let ast::Expr::Call(ast::ExprCall { func, arguments, .. }) = value else { return; }; + let func_ty = self + .try_expression_type(func) + .unwrap_or_else(|| self.infer_expression(func, TypeContext::default())); + let known_class = func_ty + .as_class_literal() + .and_then(|cls| cls.known(self.db())); + if let Some(KnownClass::NewType) = known_class { + self.infer_newtype_assignment_deferred(arguments); + return; + } for arg in arguments.args.iter().skip(1) { self.infer_type_expression(arg); } @@ -4907,12 +5012,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.infer_type_expression(&bound.value); } if let Some(default) = arguments.find_keyword("default") { - let func_ty = self - .try_expression_type(func) - .unwrap_or_else(|| self.infer_expression(func, TypeContext::default())); - if func_ty.as_class_literal().is_some_and(|class_literal| { - class_literal.is_known(self.db(), KnownClass::ParamSpec) - }) { + if let Some(KnownClass::ParamSpec) = known_class { self.infer_paramspec_default(&default.value); } else { self.infer_type_expression(&default.value); @@ -4920,6 +5020,34 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + // Infer the deferred base type of a NewType. + fn infer_newtype_assignment_deferred(&mut self, arguments: &ast::Arguments) { + match self.infer_type_expression(&arguments.args[1]) { + Type::NominalInstance(_) | Type::NewTypeInstance(_) => {} + // `Unknown` is likely to be the result of an unresolved import or a typo, which will + // already get a diagnostic, so don't pile on an extra diagnostic here. + Type::Dynamic(DynamicType::Unknown) => {} + other_type => { + if let Some(builder) = self + .context + .report_lint(&INVALID_NEWTYPE, &arguments.args[1]) + { + let mut diag = builder.into_diagnostic("invalid base for `typing.NewType`"); + diag.set_primary_message(format!("type `{}`", other_type.display(self.db()))); + if matches!(other_type, Type::ProtocolInstance(_)) { + diag.info("The base of a `NewType` is not allowed to be a protocol class."); + } else if matches!(other_type, Type::TypedDict(_)) { + diag.info("The base of a `NewType` is not allowed to be a `TypedDict`."); + } else { + diag.info( + "The base of a `NewType` must be a class type or another `NewType`.", + ); + } + } + } + } + } + fn infer_annotated_assignment_statement(&mut self, assignment: &ast::StmtAnnAssign) { if assignment.target.is_name_expr() { self.infer_definition(assignment); @@ -7483,11 +7611,11 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .to_class_type(self.db()) .is_none_or(|enum_class| !class.is_subclass_of(self.db(), enum_class)) { - // Inference of correctly-placed `TypeVar` and `ParamSpec` definitions is done in - // `TypeInferenceBuilder::infer_legacy_typevar` and - // `TypeInferenceBuilder::infer_paramspec`, and doesn't use the full - // call-binding machinery. If we reach here, it means that someone is trying to - // instantiate a `typing.TypeVar` and `typing.ParamSpec` in an invalid context. + // Inference of correctly-placed `TypeVar`, `ParamSpec`, and `NewType` definitions + // is done in `infer_legacy_typevar`, `infer_paramspec`, and + // `infer_newtype_expression`, and doesn't use the full call-binding machinery. If + // we reach here, it means that someone is trying to instantiate one of these in an + // invalid context. match class.known(self.db()) { Some(KnownClass::TypeVar | KnownClass::ExtensionsTypeVar) => { if let Some(builder) = self @@ -7509,6 +7637,15 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ); } } + Some(KnownClass::NewType) => { + if let Some(builder) = + self.context.report_lint(&INVALID_NEWTYPE, call_expression) + { + builder.into_diagnostic( + "A `NewType` definition must be a simple variable assignment", + ); + } + } _ => {} } @@ -8577,7 +8714,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | Type::BoundSuper(_) | Type::TypeVar(_) | Type::TypeIs(_) - | Type::TypedDict(_), + | Type::TypedDict(_) + | Type::NewTypeInstance(_), ) => { let unary_dunder_method = match op { ast::UnaryOp::Invert => "__invert__", @@ -9025,7 +9163,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | Type::BoundSuper(_) | Type::TypeVar(_) | Type::TypeIs(_) - | Type::TypedDict(_), + | Type::TypedDict(_) + | Type::NewTypeInstance(_), Type::FunctionLiteral(_) | Type::BooleanLiteral(_) | Type::Callable(..) @@ -9054,7 +9193,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | Type::BoundSuper(_) | Type::TypeVar(_) | Type::TypeIs(_) - | Type::TypedDict(_), + | Type::TypedDict(_) + | Type::NewTypeInstance(_), op, ) => Type::try_call_bin_op(self.db(), left_ty, op, right_ty) .map(|outcome| outcome.return_type(self.db())) diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 7694839c15c73..9fc1f35b2a85f 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -828,6 +828,16 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.infer_type_expression(slice); todo_type!("Generic specialization of typing.Annotated") } + KnownInstanceType::NewType(newtype) => { + self.infer_type_expression(&subscript.slice); + if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { + builder.into_diagnostic(format_args!( + "`{}` is a `NewType` and cannot be specialized", + newtype.name(self.db()) + )); + } + Type::unknown() + } }, Type::Dynamic(DynamicType::Todo(_)) => { self.infer_type_expression(slice); diff --git a/crates/ty_python_semantic/src/types/narrow.rs b/crates/ty_python_semantic/src/types/narrow.rs index 2e81c92448fe0..8dc6f2f6269b5 100644 --- a/crates/ty_python_semantic/src/types/narrow.rs +++ b/crates/ty_python_semantic/src/types/narrow.rs @@ -252,7 +252,8 @@ impl ClassInfoConstraintFunction { | Type::TypeIs(_) | Type::WrapperDescriptor(_) | Type::DataclassTransformer(_) - | Type::TypedDict(_) => None, + | Type::TypedDict(_) + | Type::NewTypeInstance(_) => None, } } } diff --git a/crates/ty_python_semantic/src/types/newtype.rs b/crates/ty_python_semantic/src/types/newtype.rs new file mode 100644 index 0000000000000..fe08fa7bee282 --- /dev/null +++ b/crates/ty_python_semantic/src/types/newtype.rs @@ -0,0 +1,266 @@ +use std::collections::BTreeSet; + +use crate::Db; +use crate::semantic_index::definition::{Definition, DefinitionKind}; +use crate::types::constraints::ConstraintSet; +use crate::types::{ClassType, Type, definition_expression_type, visitor}; +use ruff_db::parsed::parsed_module; +use ruff_python_ast as ast; + +/// A `typing.NewType` declaration, either from the perspective of the +/// identity-callable-that-acts-like-a-subtype-in-type-expressions returned by the call to +/// `typing.NewType(...)`, or from the perspective of instances of that subtype returned by the +/// identity callable. For example: +/// +/// ```py +/// import typing +/// Foo = typing.NewType("Foo", int) +/// x = Foo(42) +/// ``` +/// +/// The revealed types there are: +/// - `typing.NewType`: `Type::ClassLiteral(ClassLiteral)` with `KnownClass::NewType`. +/// - `Foo`: `Type::KnownInstance(KnownInstanceType::NewType(NewType { .. }))` +/// - `x`: `Type::NewTypeInstance(NewType { .. })` +/// +/// # Ordering +/// Ordering is based on the newtype's salsa-assigned id and not on its values. +/// The id may change between runs, or when the newtype was garbage collected and recreated. +#[salsa::interned(debug, heap_size=ruff_memory_usage::heap_size)] +#[derive(PartialOrd, Ord)] +pub struct NewType<'db> { + /// The name of this NewType (e.g. `"Foo"`) + #[returns(ref)] + pub name: ast::name::Name, + + /// The binding where this NewType is first created. + pub definition: Definition<'db>, + + // The base type of this NewType, if it's eagerly specified. This is typically `None` when a + // `NewType` is first encountered, because the base type is lazy/deferred to avoid panics in + // the recursive case. This becomes `Some` when a `NewType` is modified by methods like + // `.normalize()`. Callers should use the `base` method instead of accessing this field + // directly. + eager_base: Option>, +} + +impl get_size2::GetSize for NewType<'_> {} + +#[salsa::tracked] +impl<'db> NewType<'db> { + pub fn base(self, db: &'db dyn Db) -> NewTypeBase<'db> { + match self.eager_base(db) { + Some(base) => base, + None => self.lazy_base(db), + } + } + + #[salsa::tracked( + cycle_initial=lazy_base_cycle_initial, + heap_size=ruff_memory_usage::heap_size + )] + fn lazy_base(self, db: &'db dyn Db) -> NewTypeBase<'db> { + // `TypeInferenceBuilder` emits diagnostics for invalid `NewType` definitions that show up + // in assignments, but invalid definitions still get here, and also `NewType` might show up + // in places that aren't definitions at all. Fall back to `object` in all error cases. + let object_fallback = NewTypeBase::ClassType(ClassType::object(db)); + let definition = self.definition(db); + let module = parsed_module(db, definition.file(db)).load(db); + let DefinitionKind::Assignment(assignment) = definition.kind(db) else { + return object_fallback; + }; + let Some(call_expr) = assignment.value(&module).as_call_expr() else { + return object_fallback; + }; + let Some(second_arg) = call_expr.arguments.args.get(1) else { + return object_fallback; + }; + match definition_expression_type(db, definition, second_arg) { + Type::NominalInstance(nominal_instance_type) => { + NewTypeBase::ClassType(nominal_instance_type.class(db)) + } + Type::NewTypeInstance(newtype) => NewTypeBase::NewType(newtype), + // This branch includes bases that are other typing constructs besides classes and + // other newtypes, for example unions. `NewType("Foo", int | str)` is not allowed. + _ => object_fallback, + } + } + + fn iter_bases(self, db: &'db dyn Db) -> NewTypeBaseIter<'db> { + NewTypeBaseIter { + current: Some(self), + seen_before: BTreeSet::new(), + db, + } + } + + // Walk the `NewTypeBase` chain to find the underlying `ClassType`. There might not be a + // `ClassType` if this `NewType` is cyclical, and we fall back to `object` in that case. + pub fn base_class_type(self, db: &'db dyn Db) -> ClassType<'db> { + for base in self.iter_bases(db) { + if let NewTypeBase::ClassType(class_type) = base { + return class_type; + } + } + ClassType::object(db) + } + + pub(crate) fn is_equivalent_to_impl(self, db: &'db dyn Db, other: Self) -> bool { + // Two instances of the "same" `NewType` won't compare == if one of them has an eagerly + // evaluated base (or a normalized base, etc.) and the other doesn't, so we only check for + // equality of the `definition`. + self.definition(db) == other.definition(db) + } + + // Since a regular class can't inherit from a newtype, the only way for one newtype to be a + // subtype of another is to have the other in its chain of newtype bases. Once we reach the + // base class, we don't have to keep looking. + pub(crate) fn has_relation_to_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> { + if self.is_equivalent_to_impl(db, other) { + return ConstraintSet::from(true); + } + for base in self.iter_bases(db) { + if let NewTypeBase::NewType(base_newtype) = base { + if base_newtype.is_equivalent_to_impl(db, other) { + return ConstraintSet::from(true); + } + } + } + ConstraintSet::from(false) + } + + pub(crate) fn is_disjoint_from_impl(self, db: &'db dyn Db, other: Self) -> ConstraintSet<'db> { + // Two NewTypes are disjoint if they're not equal and neither inherits from the other. + // NewTypes have single inheritance, and a regular class can't inherit from a NewType, so + // it's not possible for some third type to multiply-inherit from both. + let mut self_not_subtype_of_other = self.has_relation_to_impl(db, other).negate(db); + let other_not_subtype_of_self = other.has_relation_to_impl(db, self).negate(db); + self_not_subtype_of_other.intersect(db, other_not_subtype_of_self) + } + + /// Create a new `NewType` by mapping the underlying `ClassType`. This descends through any + /// number of nested `NewType` layers and rebuilds the whole chain. In the rare case of cyclic + /// `NewType`s with no underlying `ClassType`, this has no effect and does not call `f`. + pub(crate) fn map_base_class_type( + self, + db: &'db dyn Db, + f: impl FnOnce(ClassType<'db>) -> ClassType<'db>, + ) -> Self { + // Modifying the base class type requires unwrapping and re-wrapping however many base + // newtypes there are between here and there. Normally recursion would be natural for this, + // but the bases iterator does cycle detection, and I think using that with a stack is a + // little cleaner than conjuring up yet another `CycleDetector` visitor and yet another + // layer of "*_impl" nesting. Also if there is no base class type, returning `self` + // unmodified seems more correct than injecting some default type like `object` into the + // cycle, which is what `CycleDetector` would do if we used it here. + let mut inner_newtype_stack = Vec::new(); + for base in self.iter_bases(db) { + match base { + // Build up the stack of intermediate newtypes that we'll need to re-wrap after + // we've mapped the `ClassType`. + NewTypeBase::NewType(base_newtype) => inner_newtype_stack.push(base_newtype), + // We've reached the `ClassType`. + NewTypeBase::ClassType(base_class_type) => { + // Call `f`. + let mut mapped_base = NewTypeBase::ClassType(f(base_class_type)); + // Re-wrap the mapped base class in however many newtypes we unwrapped. + for inner_newtype in inner_newtype_stack.into_iter().rev() { + mapped_base = NewTypeBase::NewType(NewType::new( + db, + inner_newtype.name(db).clone(), + inner_newtype.definition(db), + Some(mapped_base), + )); + } + return NewType::new( + db, + self.name(db).clone(), + self.definition(db), + Some(mapped_base), + ); + } + } + } + // If we get here, there is no `ClassType` (because this newtype is cyclic), and we don't + // call `f` at all. + self + } +} + +pub(crate) fn walk_newtype_instance_type<'db, V: visitor::TypeVisitor<'db> + ?Sized>( + db: &'db dyn Db, + newtype: NewType<'db>, + visitor: &V, +) { + visitor.visit_type(db, newtype.base(db).instance_type(db)); +} + +/// `typing.NewType` typically wraps a class type, but it can also wrap another newtype. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, get_size2::GetSize, salsa::Update)] +pub enum NewTypeBase<'db> { + ClassType(ClassType<'db>), + NewType(NewType<'db>), +} + +impl<'db> NewTypeBase<'db> { + pub fn instance_type(self, db: &'db dyn Db) -> Type<'db> { + match self { + NewTypeBase::ClassType(class_type) => Type::instance(db, class_type), + NewTypeBase::NewType(newtype) => Type::NewTypeInstance(newtype), + } + } +} + +/// An iterator over the transitive bases of a `NewType`. In the most common case, e.g. +/// `Foo = NewType("Foo", int)`, this yields the one `NewTypeBase::ClassType` (e.g. `int`). For +/// newtypes that wrap other newtypes, this iterator yields the `NewTypeBase::NewType`s (not +/// including `self`) before finally yielding the `NewTypeBase::ClassType`. In the pathological +/// case of cyclic newtypes like `Foo = NewType("Foo", "Foo")`, this iterator yields the unique +/// `NewTypeBase::NewType`s (not including `self`), detects the cycle, and then stops. +/// +/// Note that this does *not* detect indirect cycles that go through a proper class, like this: +/// ```py +/// Foo = NewType("Foo", list["Foo"]) +/// ``` +/// As far as this iterator is concerned, that's the "common case", and it yields the one +/// `NewTypeBase::ClassType` for `list[Foo]`. Functions like `normalize` that continue recursing +/// over the base class need to pass down a cycle-detecting visitor as usual. +struct NewTypeBaseIter<'db> { + current: Option>, + seen_before: BTreeSet>, + db: &'db dyn Db, +} + +impl<'db> Iterator for NewTypeBaseIter<'db> { + type Item = NewTypeBase<'db>; + + fn next(&mut self) -> Option { + let current = self.current?; + match current.base(self.db) { + NewTypeBase::ClassType(base_class_type) => { + self.current = None; + Some(NewTypeBase::ClassType(base_class_type)) + } + NewTypeBase::NewType(base_newtype) => { + // Doing the insertion only in this branch avoids allocating in the common case. + self.seen_before.insert(current); + if self.seen_before.contains(&base_newtype) { + // Cycle detected. Stop iterating. + self.current = None; + None + } else { + self.current = Some(base_newtype); + Some(NewTypeBase::NewType(base_newtype)) + } + } + } + } +} + +fn lazy_base_cycle_initial<'db>( + db: &'db dyn Db, + _id: salsa::Id, + _self: NewType<'db>, +) -> NewTypeBase<'db> { + NewTypeBase::ClassType(ClassType::object(db)) +} diff --git a/crates/ty_python_semantic/src/types/type_ordering.rs b/crates/ty_python_semantic/src/types/type_ordering.rs index f6797f87d939f..946b6173a2111 100644 --- a/crates/ty_python_semantic/src/types/type_ordering.rs +++ b/crates/ty_python_semantic/src/types/type_ordering.rs @@ -213,6 +213,10 @@ pub(super) fn union_or_intersection_elements_ordering<'db>( (Type::TypedDict(_), _) => Ordering::Less, (_, Type::TypedDict(_)) => Ordering::Greater, + (Type::NewTypeInstance(left), Type::NewTypeInstance(right)) => left.cmp(right), + (Type::NewTypeInstance(_), _) => Ordering::Less, + (_, Type::NewTypeInstance(_)) => Ordering::Greater, + (Type::Union(_), _) | (_, Type::Union(_)) => { unreachable!("our type representation does not permit nested unions"); } diff --git a/crates/ty_python_semantic/src/types/visitor.rs b/crates/ty_python_semantic/src/types/visitor.rs index dd1ddfdfe528e..7692c205ff086 100644 --- a/crates/ty_python_semantic/src/types/visitor.rs +++ b/crates/ty_python_semantic/src/types/visitor.rs @@ -11,6 +11,7 @@ use crate::{ class::walk_generic_alias, function::{FunctionType, walk_function_type}, instance::{walk_nominal_instance_type, walk_protocol_instance_type}, + newtype::{NewType, walk_newtype_instance_type}, subclass_of::walk_subclass_of_type, walk_bound_method_type, walk_bound_type_var_type, walk_callable_type, walk_intersection_type, walk_known_instance_type, walk_method_wrapper_type, @@ -109,6 +110,10 @@ pub(crate) trait TypeVisitor<'db> { fn visit_typed_dict_type(&self, db: &'db dyn Db, typed_dict: TypedDictType<'db>) { walk_typed_dict_type(db, typed_dict, self); } + + fn visit_newtype_instance_type(&self, db: &'db dyn Db, newtype: NewType<'db>) { + walk_newtype_instance_type(db, newtype, self); + } } /// Enumeration of types that may contain other types, such as unions, intersections, and generics. @@ -131,6 +136,7 @@ pub(super) enum NonAtomicType<'db> { ProtocolInstance(ProtocolInstanceType<'db>), TypedDict(TypedDictType<'db>), TypeAlias(TypeAliasType<'db>), + NewTypeInstance(NewType<'db>), } pub(super) enum TypeKind<'db> { @@ -198,6 +204,9 @@ impl<'db> From> for TypeKind<'db> { TypeKind::NonAtomic(NonAtomicType::TypedDict(typed_dict)) } Type::TypeAlias(alias) => TypeKind::NonAtomic(NonAtomicType::TypeAlias(alias)), + Type::NewTypeInstance(newtype) => { + TypeKind::NonAtomic(NonAtomicType::NewTypeInstance(newtype)) + } } } } @@ -239,6 +248,9 @@ pub(super) fn walk_non_atomic_type<'db, V: TypeVisitor<'db> + ?Sized>( NonAtomicType::TypeAlias(alias) => { visitor.visit_type_alias_type(db, alias); } + NonAtomicType::NewTypeInstance(newtype) => { + visitor.visit_newtype_instance_type(db, newtype); + } } } diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index 7373c4cf25ed7..ab96ebf4e85d3 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -59,6 +59,7 @@ Settings: Settings { "invalid-legacy-type-variable": Error (Default), "invalid-metaclass": Error (Default), "invalid-named-tuple": Error (Default), + "invalid-newtype": Error (Default), "invalid-overload": Error (Default), "invalid-parameter-default": Error (Default), "invalid-paramspec": Error (Default), diff --git a/ty.schema.json b/ty.schema.json index cae55e4a1b29e..3026102a5f14b 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -623,6 +623,16 @@ } ] }, + "invalid-newtype": { + "title": "detects invalid NewType definitions", + "description": "## What it does\nChecks for the creation of invalid `NewType`s\n\n## Why is this bad?\nThere are several requirements that you must follow when creating a `NewType`.\n\n## Examples\n```python\nfrom typing import NewType\n\ndef get_name() -> str: ...\n\nFoo = NewType(\"Foo\", int) # okay\nBar = NewType(get_name(), int) # error: The first argument to `NewType` must be a string literal\nBaz = NewType(\"Baz\", int | str) # error: invalid base for `typing.NewType`\n```", + "default": "error", + "oneOf": [ + { + "$ref": "#/definitions/Level" + } + ] + }, "invalid-overload": { "title": "detects invalid `@overload` usages", "description": "## What it does\nChecks for various invalid `@overload` usages.\n\n## Why is this bad?\nThe `@overload` decorator is used to define functions and methods that accepts different\ncombinations of arguments and return different types based on the arguments passed. This is\nmainly beneficial for type checkers. But, if the `@overload` usage is invalid, the type\nchecker may not be able to provide correct type information.\n\n## Example\n\nDefining only one overload:\n\n```py\nfrom typing import overload\n\n@overload\ndef foo(x: int) -> int: ...\ndef foo(x: int | None) -> int | None:\n return x\n```\n\nOr, not providing an implementation for the overloaded definition:\n\n```py\nfrom typing import overload\n\n@overload\ndef foo() -> None: ...\n@overload\ndef foo(x: int) -> int: ...\n```\n\n## References\n- [Python documentation: `@overload`](https://docs.python.org/3/library/typing.html#typing.overload)",