From 115dbf52c3d7d036e77f16af3e8ac90c6095fbbe Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 6 May 2022 15:58:41 +0100 Subject: [PATCH 1/8] Improve `csv` module stubs --- stdlib/_csv.pyi | 51 ++++++++++++++++++++++++++++++++++++++++--------- stdlib/csv.pyi | 34 +++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/stdlib/_csv.pyi b/stdlib/_csv.pyi index ae9031df6e81..a5b2b7fd6363 100644 --- a/stdlib/_csv.pyi +++ b/stdlib/_csv.pyi @@ -1,5 +1,6 @@ +from _typeshed import SupportsWrite from collections.abc import Iterable, Iterator -from typing import Any, Protocol, Union +from typing import Any, Union from typing_extensions import Literal, TypeAlias __version__: str @@ -25,21 +26,53 @@ class Dialect: _DialectLike: TypeAlias = Union[str, Dialect, type[Dialect]] class _reader(Iterator[list[str]]): - dialect: Dialect + @property + def dialect(self) -> Dialect: ... line_num: int def __next__(self) -> list[str]: ... class _writer: - dialect: Dialect + @property + def dialect(self) -> Dialect: ... def writerow(self, row: Iterable[Any]) -> Any: ... def writerows(self, rows: Iterable[Iterable[Any]]) -> None: ... -class _Writer(Protocol): - def write(self, __s: str) -> object: ... - -def writer(csvfile: _Writer, dialect: _DialectLike = ..., **fmtparams: Any) -> _writer: ... -def reader(csvfile: Iterable[str], dialect: _DialectLike = ..., **fmtparams: Any) -> _reader: ... -def register_dialect(name: str, dialect: Any = ..., **fmtparams: Any) -> None: ... +def writer( + csvfile: SupportsWrite[str], + dialect: _DialectLike = ..., + *, + delimiter: str = ..., + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... +) -> _writer: ... +def reader( + csvfile: Iterable[str], + dialect: _DialectLike = ..., + *, + delimiter: str = ..., + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... +) -> _reader: ... +def register_dialect( + name: str, + dialect: Any = ..., + *, + delimiter: str = ..., + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... +) -> None: ... def unregister_dialect(name: str) -> None: ... def get_dialect(name: str) -> Dialect: ... def list_dialects() -> list[str]: ... diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index dcb3f19bebe1..1e31d40eee78 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -4,6 +4,8 @@ from _csv import ( QUOTE_MINIMAL as QUOTE_MINIMAL, QUOTE_NONE as QUOTE_NONE, QUOTE_NONNUMERIC as QUOTE_NONNUMERIC, + # actually csv.Dialect is a different class to _csv.Dialect at runtime, + # but for typing purposes, they're identical Dialect as Dialect, Error as Error, __version__ as __version__, @@ -18,7 +20,7 @@ from _csv import ( unregister_dialect as unregister_dialect, writer as writer, ) -from _typeshed import Self +from _typeshed import Self, SupportsWrite from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence from typing import Any, Generic, TypeVar, overload @@ -87,8 +89,14 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): restkey: str | None = ..., restval: str | None = ..., dialect: _DialectLike = ..., - *args: Any, - **kwds: Any, + *, + delimiter: str = ..., + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... ) -> None: ... @overload def __init__( @@ -98,8 +106,13 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): restkey: str | None = ..., restval: str | None = ..., dialect: _DialectLike = ..., - *args: Any, - **kwds: Any, + *, + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... ) -> None: ... def __iter__(self: Self) -> Self: ... def __next__(self) -> _DictReadMapping[_T, str]: ... @@ -111,13 +124,18 @@ class DictWriter(Generic[_T]): writer: _writer def __init__( self, - f: Any, + f: SupportsWrite[str], fieldnames: Collection[_T], restval: Any | None = ..., extrasaction: str = ..., dialect: _DialectLike = ..., - *args: Any, - **kwds: Any, + *, + quotechar: str | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool = ..., + lineterminator: str = ..., + quoting: int = ..., + strict: int = ... ) -> None: ... if sys.version_info >= (3, 8): def writeheader(self) -> Any: ... From 13e00d5196c6eb3f944a3ff5e4ed219ecc29b093 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 15:00:28 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/_csv.pyi | 6 +++--- stdlib/csv.pyi | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/stdlib/_csv.pyi b/stdlib/_csv.pyi index a5b2b7fd6363..636e0ad45fad 100644 --- a/stdlib/_csv.pyi +++ b/stdlib/_csv.pyi @@ -47,7 +47,7 @@ def writer( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> _writer: ... def reader( csvfile: Iterable[str], @@ -59,7 +59,7 @@ def reader( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> _reader: ... def register_dialect( name: str, @@ -71,7 +71,7 @@ def register_dialect( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> None: ... def unregister_dialect(name: str) -> None: ... def get_dialect(name: str) -> Dialect: ... diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 1e31d40eee78..12944bc2ca14 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -1,11 +1,9 @@ import sys -from _csv import ( +from _csv import ( # actually csv.Dialect is a different class to _csv.Dialect at runtime,; but for typing purposes, they're identical QUOTE_ALL as QUOTE_ALL, QUOTE_MINIMAL as QUOTE_MINIMAL, QUOTE_NONE as QUOTE_NONE, QUOTE_NONNUMERIC as QUOTE_NONNUMERIC, - # actually csv.Dialect is a different class to _csv.Dialect at runtime, - # but for typing purposes, they're identical Dialect as Dialect, Error as Error, __version__ as __version__, @@ -96,7 +94,7 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> None: ... @overload def __init__( @@ -112,7 +110,7 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> None: ... def __iter__(self: Self) -> Self: ... def __next__(self) -> _DictReadMapping[_T, str]: ... @@ -135,7 +133,7 @@ class DictWriter(Generic[_T]): skipinitialspace: bool = ..., lineterminator: str = ..., quoting: int = ..., - strict: int = ... + strict: int = ..., ) -> None: ... if sys.version_info >= (3, 8): def writeheader(self) -> Any: ... From 26d52061fa7e2ab021cb00ab9c86bfb7c043423e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 6 May 2022 16:01:49 +0100 Subject: [PATCH 3/8] Update stdlib/csv.pyi --- stdlib/csv.pyi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 12944bc2ca14..86e8e31b7fbc 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -1,5 +1,6 @@ import sys -from _csv import ( # actually csv.Dialect is a different class to _csv.Dialect at runtime,; but for typing purposes, they're identical +# actually csv.Dialect is a different class to _csv.Dialect at runtime, but for typing purposes, they're identical +from _csv import ( QUOTE_ALL as QUOTE_ALL, QUOTE_MINIMAL as QUOTE_MINIMAL, QUOTE_NONE as QUOTE_NONE, From a76d2878db907575e45bc3a5ffbcd9d291102afa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 May 2022 15:03:01 +0000 Subject: [PATCH 4/8] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/csv.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 86e8e31b7fbc..402a64bf1f5f 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -1,4 +1,5 @@ import sys + # actually csv.Dialect is a different class to _csv.Dialect at runtime, but for typing purposes, they're identical from _csv import ( QUOTE_ALL as QUOTE_ALL, From dffbb17dfd7f530e579494db5a84125ba0f560e3 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 6 May 2022 16:16:53 +0100 Subject: [PATCH 5/8] `extrasaction` can only be `Literal['raise', 'ignore']`. Fix stubtest also. --- stdlib/csv.pyi | 5 +++-- tests/stubtest_allowlists/py3_common.txt | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 402a64bf1f5f..bfbfb8ef0272 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -23,6 +23,7 @@ from _csv import ( from _typeshed import Self, SupportsWrite from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence from typing import Any, Generic, TypeVar, overload +from typing_extensions import Literal if sys.version_info >= (3, 8): from builtins import dict as _DictReadMapping @@ -120,14 +121,14 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): class DictWriter(Generic[_T]): fieldnames: Collection[_T] restval: Any | None - extrasaction: str + extrasaction: Literal["raise", "ignore"] writer: _writer def __init__( self, f: SupportsWrite[str], fieldnames: Collection[_T], restval: Any | None = ..., - extrasaction: str = ..., + extrasaction: Literal["raise", "ignore"] = ..., dialect: _DialectLike = ..., *, quotechar: str | None = ..., diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index 687aa32b53a3..89f80407514d 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -78,6 +78,8 @@ csv.Dialect.doublequote csv.Dialect.lineterminator csv.Dialect.quoting csv.Dialect.skipinitialspace +csv.DictReader.__init__ # runtime sig has *args but will error if more than 5 positional args are supplied +csv.DictWriter.__init__ # runtime sig has *args but will error if more than 5 positional args are supplied ctypes.Array.__iter__ # mypy doesn't support using __getitem__ instead of __iter__ so this is here https://github.com/python/mypy/issues/2220 ctypes.Array._type_ # _type_ and _length_ are abstract, https://github.com/python/typeshed/pull/6361 ctypes.Array._length_ From 0bb97c945ca5f82e585b5667b0be361f39862b35 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 6 May 2022 16:35:48 +0100 Subject: [PATCH 6/8] Add back `doublequote`, add comment regarding quoting --- stdlib/_csv.pyi | 15 +++++++++++---- stdlib/csv.pyi | 10 +++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stdlib/_csv.pyi b/stdlib/_csv.pyi index 636e0ad45fad..613616348ea9 100644 --- a/stdlib/_csv.pyi +++ b/stdlib/_csv.pyi @@ -10,6 +10,10 @@ QUOTE_MINIMAL: Literal[0] QUOTE_NONE: Literal[3] QUOTE_NONNUMERIC: Literal[2] +# Ideally this would be Union[QUOTE_ALL, QUOTE_MINIMAL, QUOTE_NONE, QUOTE_NONNUMERIC] +# However, using literals in situations like these can cause false-positives (see #7258) +_QuotingType: TypeAlias = int + class Error(Exception): ... class Dialect: @@ -19,7 +23,7 @@ class Dialect: doublequote: bool skipinitialspace: bool lineterminator: str - quoting: int + quoting: _QuotingType strict: int def __init__(self) -> None: ... @@ -44,9 +48,10 @@ def writer( delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> _writer: ... def reader( @@ -56,9 +61,10 @@ def reader( delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> _reader: ... def register_dialect( @@ -68,9 +74,10 @@ def register_dialect( delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> None: ... def unregister_dialect(name: str) -> None: ... diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index bfbfb8ef0272..9e8d8226333c 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -10,6 +10,7 @@ from _csv import ( Error as Error, __version__ as __version__, _DialectLike, + _QuotingType, _reader, _writer, field_size_limit as field_size_limit, @@ -94,9 +95,10 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> None: ... @overload @@ -110,9 +112,10 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): *, quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> None: ... def __iter__(self: Self) -> Self: ... @@ -133,9 +136,10 @@ class DictWriter(Generic[_T]): *, quotechar: str | None = ..., escapechar: str | None = ..., + doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., - quoting: int = ..., + quoting: _QuotingType = ..., strict: int = ..., ) -> None: ... if sys.version_info >= (3, 8): From ed73fa179afc67e18c703d90c1f3fbe3ce9a74b2 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 6 May 2022 16:56:26 +0100 Subject: [PATCH 7/8] `strict` should be `bool`, add back `delimiter` argument --- stdlib/_csv.pyi | 8 ++++---- stdlib/csv.pyi | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/stdlib/_csv.pyi b/stdlib/_csv.pyi index 613616348ea9..910d49be027f 100644 --- a/stdlib/_csv.pyi +++ b/stdlib/_csv.pyi @@ -24,7 +24,7 @@ class Dialect: skipinitialspace: bool lineterminator: str quoting: _QuotingType - strict: int + strict: bool def __init__(self) -> None: ... _DialectLike: TypeAlias = Union[str, Dialect, type[Dialect]] @@ -52,7 +52,7 @@ def writer( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> _writer: ... def reader( csvfile: Iterable[str], @@ -65,7 +65,7 @@ def reader( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> _reader: ... def register_dialect( name: str, @@ -78,7 +78,7 @@ def register_dialect( skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> None: ... def unregister_dialect(name: str) -> None: ... def get_dialect(name: str) -> Dialect: ... diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 9e8d8226333c..5ee049ed1427 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -63,7 +63,7 @@ class excel(Dialect): doublequote: bool skipinitialspace: bool lineterminator: str - quoting: int + quoting: _QuotingType class excel_tab(excel): delimiter: str @@ -74,7 +74,7 @@ class unix_dialect(Dialect): doublequote: bool skipinitialspace: bool lineterminator: str - quoting: int + quoting: _QuotingType class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): fieldnames: Sequence[_T] | None @@ -99,7 +99,7 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> None: ... @overload def __init__( @@ -110,13 +110,14 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): restval: str | None = ..., dialect: _DialectLike = ..., *, + delimiter: str = ... quotechar: str | None = ..., escapechar: str | None = ..., doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> None: ... def __iter__(self: Self) -> Self: ... def __next__(self) -> _DictReadMapping[_T, str]: ... @@ -134,13 +135,14 @@ class DictWriter(Generic[_T]): extrasaction: Literal["raise", "ignore"] = ..., dialect: _DialectLike = ..., *, + delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., doublequote: bool = ..., skipinitialspace: bool = ..., lineterminator: str = ..., quoting: _QuotingType = ..., - strict: int = ..., + strict: bool = ..., ) -> None: ... if sys.version_info >= (3, 8): def writeheader(self) -> Any: ... From 892197b271a55bff2822d8a3a53de9dd878bb2a3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 6 May 2022 16:58:06 +0100 Subject: [PATCH 8/8] Update stdlib/csv.pyi --- stdlib/csv.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/csv.pyi b/stdlib/csv.pyi index 5ee049ed1427..de69c71ad941 100644 --- a/stdlib/csv.pyi +++ b/stdlib/csv.pyi @@ -110,7 +110,7 @@ class DictReader(Generic[_T], Iterator[_DictReadMapping[_T, str]]): restval: str | None = ..., dialect: _DialectLike = ..., *, - delimiter: str = ... + delimiter: str = ..., quotechar: str | None = ..., escapechar: str | None = ..., doublequote: bool = ...,