From dde12809d6e541f25de7bcc6a76c94bd6089475f Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Thu, 9 Jan 2020 17:05:27 -0800 Subject: [PATCH 1/4] Fix some crashes in dataclasses Store the type of an attribute in `DataclassAttribute` so we don't need to pull it out of `TypeInfo` in a fragile way (since the child might try to override it in a way that breaks things). This also allows us to get rid of some pretty dodgy code having to do with InitVars (that could cause an incremental mode crash.) Fixes #6809 and an unreported incremental bug --- mypy/plugins/common.py | 19 +++++-- mypy/plugins/dataclasses.py | 71 ++++++++++--------------- test-data/unit/check-attr.test | 15 ++++++ test-data/unit/check-dataclasses.test | 75 +++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 49 deletions(-) diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 6f2f5845cbeb4..66105f96a3eb5 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -1,15 +1,18 @@ -from typing import List, Optional +from typing import List, Optional, Union from mypy.nodes import ( ARG_POS, MDEF, Argument, Block, CallExpr, Expression, SYMBOL_FUNCBASE_TYPES, - FuncDef, PassStmt, RefExpr, SymbolTableNode, Var + FuncDef, PassStmt, RefExpr, SymbolTableNode, Var, JsonDict, ) -from mypy.plugin import ClassDefContext +from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface from mypy.semanal import set_callable_name -from mypy.types import CallableType, Overloaded, Type, TypeVarDef, get_proper_type +from mypy.types import ( + CallableType, Overloaded, Type, TypeVarDef, deserialize_type, get_proper_type, +) from mypy.typevars import fill_typevars from mypy.util import get_unique_redefinition_name from mypy.typeops import try_getting_str_literals # noqa: F401 # Part of public API +from mypy.fixup import TypeFixer def _get_decorator_bool_argument( @@ -128,3 +131,11 @@ def add_method( info.names[name] = SymbolTableNode(MDEF, func, plugin_generated=True) info.defn.defs.body.append(func) + + +def deserialize_and_fixup_type( + data: Union[str, JsonDict], api: SemanticAnalyzerPluginInterface +) -> Type: + typ = deserialize_type(data) + typ.accept(TypeFixer(api.modules, allow_missing=False)) + return typ diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 86b24f0a58fb0..475ed3fa9b06e 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -5,12 +5,14 @@ from mypy.nodes import ( ARG_OPT, ARG_POS, MDEF, Argument, AssignmentStmt, CallExpr, - Context, Expression, FuncDef, JsonDict, NameExpr, RefExpr, + Context, Expression, JsonDict, NameExpr, RefExpr, SymbolTableNode, TempNode, TypeInfo, Var, TypeVarExpr, PlaceholderNode ) -from mypy.plugin import ClassDefContext -from mypy.plugins.common import add_method, _get_decorator_bool_argument -from mypy.types import Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type +from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface +from mypy.plugins.common import ( + add_method, _get_decorator_bool_argument, deserialize_and_fixup_type, +) +from mypy.types import Type, Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type from mypy.server.trigger import make_wildcard_trigger # The set of decorators that generate dataclasses. @@ -31,6 +33,7 @@ def __init__( has_default: bool, line: int, column: int, + type: Optional[Type], ) -> None: self.name = name self.is_in_init = is_in_init @@ -38,17 +41,18 @@ def __init__( self.has_default = has_default self.line = line self.column = column + self.type = type - def to_argument(self, info: TypeInfo) -> Argument: + def to_argument(self) -> Argument: return Argument( - variable=self.to_var(info), - type_annotation=info[self.name].type, + variable=self.to_var(), + type_annotation=self.type, initializer=None, kind=ARG_OPT if self.has_default else ARG_POS, ) - def to_var(self, info: TypeInfo) -> Var: - return Var(self.name, info[self.name].type) + def to_var(self) -> Var: + return Var(self.name, self.type) def serialize(self) -> JsonDict: return { @@ -58,11 +62,16 @@ def serialize(self) -> JsonDict: 'has_default': self.has_default, 'line': self.line, 'column': self.column, + 'type': self.type.serialize() if self.type else None, } @classmethod - def deserialize(cls, info: TypeInfo, data: JsonDict) -> 'DataclassAttribute': - return cls(**data) + def deserialize( + cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface + ) -> 'DataclassAttribute': + typ_serialized = data.pop('type') + typ = deserialize_and_fixup_type(typ_serialized, api) if typ_serialized else None + return cls(type=typ, **data) class DataclassTransformer: @@ -81,12 +90,7 @@ def transform(self) -> None: # Some definitions are not ready, defer() should be already called. return for attr in attributes: - node = info.get(attr.name) - if node is None: - # Nodes of superclass InitVars not used in __init__ cannot be reached. - assert attr.is_init_var and not attr.is_in_init - continue - if node.type is None: + if attr.type is None: ctx.api.defer() return decorator_arguments = { @@ -106,7 +110,7 @@ def transform(self) -> None: add_method( ctx, '__init__', - args=[attr.to_argument(info) for attr in attributes if attr.is_in_init], + args=[attr.to_argument() for attr in attributes if attr.is_in_init], return_type=NoneType(), ) @@ -191,7 +195,7 @@ def reset_init_only_vars(self, info: TypeInfo, attributes: List[DataclassAttribu del info.names[attr.name] else: # Nodes of superclass InitVars not used in __init__ cannot be reached. - assert attr.is_init_var and not attr.is_in_init + assert attr.is_init_var for stmt in info.defn.defs.body: if isinstance(stmt, AssignmentStmt) and stmt.unanalyzed_type: lvalue = stmt.lvalues[0] @@ -282,6 +286,7 @@ def collect_attributes(self) -> Optional[List[DataclassAttribute]]: has_default=has_default, line=stmt.line, column=stmt.column, + type=cls.info[lhs.name].type, )) # Next, collect attributes belonging to any class in the MRO @@ -301,23 +306,7 @@ def collect_attributes(self) -> Optional[List[DataclassAttribute]]: for data in info.metadata['dataclass']['attributes']: name = data['name'] # type: str if name not in known_attrs: - attr = DataclassAttribute.deserialize(info, data) - if attr.is_init_var: - # InitVars are removed from classes so, in order for them to be inherited - # properly, we need to re-inject them into subclasses' sym tables here. - # To do that, we look 'em up from the parents' __init__. These variables - # are subsequently removed from the sym table at the end of - # DataclassTransformer.transform. - superclass_init = info.get_method('__init__') - if isinstance(superclass_init, FuncDef): - attr_node = _get_arg_from_init(superclass_init, attr.name) - if attr_node is None: - # Continue the loop: we will look it up in the next MRO entry. - # Don't add it to the known or super attrs because we don't know - # anything about it yet - continue - else: - cls.info.names[attr.name] = attr_node + attr = DataclassAttribute.deserialize(info, data, ctx.api) known_attrs.add(name) super_attrs.append(attr) elif all_attrs: @@ -365,21 +354,13 @@ def _freeze(self, attributes: List[DataclassAttribute]) -> None: assert isinstance(var, Var) var.is_property = True else: - var = attr.to_var(info) + var = attr.to_var() var.info = info var.is_property = True var._fullname = info.fullname + '.' + var.name info.names[var.name] = SymbolTableNode(MDEF, var) -def _get_arg_from_init(init_method: FuncDef, attr_name: str) -> Optional[SymbolTableNode]: - """Given an init method and an attribute name, find the Var in the init method's args.""" - for arg, arg_name in zip(init_method.arguments, init_method.arg_names): - if arg_name == attr_name: - return SymbolTableNode(MDEF, arg.variable) - return None - - def dataclass_class_maker_callback(ctx: ClassDefContext) -> None: """Hooks into the class typechecking process to add support for dataclasses. """ diff --git a/test-data/unit/check-attr.test b/test-data/unit/check-attr.test index 9748ab8942bc1..fd6fc2b3755ef 100644 --- a/test-data/unit/check-attr.test +++ b/test-data/unit/check-attr.test @@ -1244,3 +1244,18 @@ class A2: b: int = attr.ib() [builtins fixtures/list.pyi] + +[case testAttrsInheritanceNoAnnotation] +import attr + +@attr.s +class A: + foo = attr.ib() # type: int + +x = 0 +@attr.s +class B(A): + foo = x + +reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' +[builtins fixtures/bool.pyi] diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 36b62d53dbf1e..97bb9954ec293 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -723,6 +723,51 @@ C(1, 'a') # E: Argument 2 to "C" has incompatible type "str"; expected "int" [builtins fixtures/primitives.pyi] +[case testDataclassesInitVarIncremental] +import a + +[file a.py] +import dataclasses +from b import A + +@dataclasses.dataclass +class B(A): + b: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + super().__post_init__(a) + self.b = a + 1 + +[file a.py.2] +import dataclasses +from b import A + +@dataclasses.dataclass +class B(A): + b: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + super().__post_init__(a) + self.b = a + 2 + +reveal_type(B) + +[file b.py] +import dataclasses + +@dataclasses.dataclass +class A: + a: dataclasses.InitVar[int] + _a: int = dataclasses.field(init=False) + + def __post_init__(self, a: int) -> None: + self._a = a +[out2] +tmp/a.py:12: note: Revealed type is 'def (a: builtins.int) -> a.B' + +[builtins fixtures/primitives.pyi] + + [case testNoComplainFieldNone] # flags: --python-version 3.6 # flags: --no-strict-optional @@ -957,3 +1002,33 @@ class A: # E: Name 'x' already defined (possibly by an import) @dataclass class B(A): pass + +[case testDataclassInheritanceNoAnnotation] +from dataclasses import dataclass + +@dataclass +class A: + foo: int + +x = 0 +@dataclass +class B(A): + foo = x + +reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' + +[case testDataclassInheritanceNoAnnotation2] +from dataclasses import dataclass + +@dataclass(frozen=True) +class A: + foo: int + +@dataclass +class B(A): + @property + def foo(self) -> int: pass # E: Signature of "foo" incompatible with supertype "A" + +reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' + +[builtins fixtures/property.pyi] From 989a6a8c0e4f1d1299a68014723da63dbd58471d Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Fri, 10 Jan 2020 14:13:22 -0800 Subject: [PATCH 2/4] fix an issue --- mypy/plugins/dataclasses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 475ed3fa9b06e..553fdec8f16de 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -69,6 +69,7 @@ def serialize(self) -> JsonDict: def deserialize( cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface ) -> 'DataclassAttribute': + data = data.copy() typ_serialized = data.pop('type') typ = deserialize_and_fixup_type(typ_serialized, api) if typ_serialized else None return cls(type=typ, **data) From 15a98923390ec9cdaa4b22fe3b88c8483fa6877f Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 15 Jan 2020 15:21:22 -0800 Subject: [PATCH 3/4] one cleanup --- mypy/plugins/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 553fdec8f16de..1e83bdec5fac6 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -287,7 +287,7 @@ def collect_attributes(self) -> Optional[List[DataclassAttribute]]: has_default=has_default, line=stmt.line, column=stmt.column, - type=cls.info[lhs.name].type, + type=sym.type, )) # Next, collect attributes belonging to any class in the MRO From acfce661e95d3a61b7a590831d7080eb706d3b83 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Wed, 15 Jan 2020 15:26:11 -0800 Subject: [PATCH 4/4] only serialize non-None types --- mypy/plugins/dataclasses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 1e83bdec5fac6..81f50d1a4c4d8 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -55,6 +55,7 @@ def to_var(self) -> Var: return Var(self.name, self.type) def serialize(self) -> JsonDict: + assert self.type return { 'name': self.name, 'is_in_init': self.is_in_init, @@ -62,7 +63,7 @@ def serialize(self) -> JsonDict: 'has_default': self.has_default, 'line': self.line, 'column': self.column, - 'type': self.type.serialize() if self.type else None, + 'type': self.type.serialize(), } @classmethod @@ -70,8 +71,7 @@ def deserialize( cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface ) -> 'DataclassAttribute': data = data.copy() - typ_serialized = data.pop('type') - typ = deserialize_and_fixup_type(typ_serialized, api) if typ_serialized else None + typ = deserialize_and_fixup_type(data.pop('type'), api) return cls(type=typ, **data)