gh-153401: Highlight matching delimiters in the REPL

Doc/tutorial/appendix.rst

@@ -17,7 +17,11 @@ control capabilities.
 Since Python 3.13, a new interactive shell is used by default.
 This one supports color, multiline editing, history browsing, and
 paste mode.  To disable color, see :ref:`using-on-controlling-color` for
-details.  Function keys provide some additional functionality.
+details.  When the cursor touches a parenthesis, bracket, brace, or string
+quote, its matching delimiter is highlighted.  This includes triple quotes
+and expressions in formatted and template strings.  Delimiters within
+string contents and comments are ignored.  Function keys provide some
+additional functionality.
 :kbd:`F1` enters the interactive help browser :mod:`pydoc`.
 :kbd:`F2` allows for browsing command-line history with neither output nor the
 :term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which

Lib/_colorize.py

@@ -412,6 +412,7 @@ class Syntax(ThemeSection):
     number: str = ANSIColors.YELLOW
     op: str = ANSIColors.RESET
     definition: str = ANSIColors.BOLD
+    matching_delimiter: str = ANSIColors.INTENSE_BACKGROUND_BLACK
     soft_keyword: str = ANSIColors.BOLD_BLUE
     reset: str = ANSIColors.RESET
 

Lib/_pyrepl/reader.py

@@ -37,7 +37,10 @@
 from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines
 from .layout import wrapped_row_end
 from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay
-from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors
+from .utils import (
+    ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef,
+    find_matching_delimiter, gen_colors,
+)
 from .trace import trace
 
 
@@ -304,6 +307,7 @@ class Reader:
     cxy: CursorXY = field(init=False)
     lxy: CursorXY = field(init=False)
     scheduled_commands: list[CommandName] = field(default_factory=list)
+    _matching_delimiter: tuple[int, int] | None = field(init=False)
     can_colorize: bool = False
     gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors
     threading_hook: Callback | None = None
@@ -386,6 +390,7 @@ def __post_init__(self) -> None:
         self.input_trans = input.KeymapTranslator(
             self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
         )
+        self._matching_delimiter = None
         self.layout = LayoutMap.empty()
         self.cxy = self.pos2xy()
         self.lxy = (self.pos, 0)
@@ -429,9 +434,8 @@ def calc_screen(self) -> RenderedScreen:
                     self,
                     reuse_full=True,
                 )
-                assert not self.last_refresh_cache.line_end_offsets or (
-                    self.last_refresh_cache.line_end_offsets[-1] >= len(self.buffer)
-                ), "Buffer modified without invalidate_buffer() call"
+                # A short screen can cache only a prefix of the buffer.
+                # get_cached_location() then falls back to a full rebuild.
             else:
                 offset, num_common_lines = self.last_refresh_cache.get_cached_location(
                     self,
@@ -602,8 +606,51 @@ def _render_message_lines(self) -> tuple[RenderLine, ...]:
     def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]:
         return ()
 
+    def _get_matching_delimiter(self) -> tuple[int, int] | None:
+        if not self.can_colorize or self.finished:
+            return None
+        if not any(
+            0 <= i < len(self.buffer) and self.buffer[i] in "()[]{}\"'"
+            for i in (self.pos, self.pos - 1)
+        ):
+            return None
+        return find_matching_delimiter(self.get_unicode(), self.pos)
+
+    def _matching_delimiter_overlays(
+        self, base_screen: RenderedScreen
+    ) -> tuple[ScreenOverlay, ...]:
+        if (
+            self._matching_delimiter is None
+            or not self.last_refresh_cache.line_end_offsets
+        ):
+            return ()
+
+        changed_rows: dict[int, list[RenderCell]] = {}
+        start, end = self._matching_delimiter
+        for offset in range(start, end):
+            # pos_to_xy() clamps positions beyond the cached layout.
+            if offset >= self.last_refresh_cache.line_end_offsets[-1]:
+                break
+            # The delimiter's end disambiguates a wrap boundary.
+            x, y = self.layout.pos_to_xy(offset + 1)
+            cells = changed_rows.setdefault(y, list(base_screen.lines[y].cells))
+            column = 0
+            for i, cell in enumerate(cells):
+                if column == x - 1 and cell.width:
+                    cells[i] = replace(
+                        cell, style=StyleRef.from_tag("matching_delimiter")
+                    )
+                    break
+                column += cell.width
+        return tuple(
+            ScreenOverlay(y, (RenderLine.from_cells(cells),))
+            for y, cells in changed_rows.items()
+        )
+
     def compose_rendered_screen(self, base_screen: RenderedScreen) -> RenderedScreen:
-        overlays = list(self.get_screen_overlays())
+        self._matching_delimiter = self._get_matching_delimiter()
+        overlays = list(self._matching_delimiter_overlays(base_screen))
+        overlays.extend(self.get_screen_overlays())
         message_lines = self._render_message_lines()
         if message_lines:
             overlays.append(ScreenOverlay(len(base_screen.lines), message_lines))
@@ -826,6 +873,7 @@ def prepare(self) -> None:
             self.finished = False
             del self.buffer[:]
             self.pos = 0
+            self._matching_delimiter = None
             self.layout = LayoutMap.empty()
             self.cxy = self.pos2xy()
             self.lxy = (self.pos, 0)
@@ -892,6 +940,11 @@ def error(self, msg: str = "none") -> None:
         self.invalidate_message()
 
     def update_screen(self) -> None:
+        if (
+            not self.invalidation.needs_screen_refresh
+            and self._get_matching_delimiter() != self._matching_delimiter
+        ):
+            self.invalidate_overlay()
         if self.invalidation.is_cursor_only:
             self.update_cursor()
             self.clear_invalidation()
@@ -939,12 +992,12 @@ def do_cmd(self, cmd: CommandInput) -> None:
             and not self.invalidation.is_cursor_only
         ):
             self.invalidate_cursor()
+        self.finished = bool(command.finish)
         self.update_screen()
 
         if command_type is not commands.digit_arg:
             self.last_command = command_type
 
-        self.finished = bool(command.finish)
         if self.finished:
             self.console.finish()
             self.finish()

Lib/_pyrepl/simple_interact.py

@@ -159,6 +159,7 @@ def maybe_run_command(statement: str) -> bool:
             r.cmpltn_reset()
             if r.input_trans is r.isearch_trans:
                 r.do_cmd(("isearch-end", [""]))
+            r.finished = True
             r.pos = len(r.get_unicode())
             r.invalidate_full()
             r.refresh()

Lib/_pyrepl/utils.py

@@ -116,12 +116,7 @@ def unbracket(s: str, including_content: bool = False) -> str:
     return s.translate(ZERO_WIDTH_TRANS)
 
 
-def gen_colors(buffer: str) -> Iterator[ColorSpan]:
-    """Returns a list of index spans to color using the given color tag.
-
-    The input `buffer` should be a valid start of a Python code block, i.e.
-    it cannot be a block starting in the middle of a multiline string.
-    """
+def _get_token_stream(buffer: str) -> tuple[Iterator[TI], list[int]]:
     sio = StringIO(buffer)
     line_lengths = [0] + [len(line) for line in sio.readlines()]
     # make line_lengths cumulative
@@ -130,6 +125,87 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
 
     sio.seek(0)
     gen = tokenize.generate_tokens(sio.readline)
+    return gen, line_lengths
+
+
+def find_matching_delimiter(buffer: str, pos: int) -> tuple[int, int] | None:
+    """Return the half-open span of the partner touching the cursor."""
+    # A delimiter under the cursor takes precedence, even inside literal data.
+    candidate = next((
+        i for i in (pos, pos - 1)
+        if 0 <= i < len(buffer) and buffer[i] in "()[]{}\"'"
+    ), None)
+    if candidate is None:
+        return None
+
+    gen, line_lengths = _get_token_stream(buffer)
+    try:
+        return _find_matching_delimiter_from_token_stream(
+            gen, line_lengths, candidate
+        )
+    except SyntaxError, tokenize.TokenError:
+        # Incomplete input can still contain complete pairs.
+        return None
+
+
+def _find_matching_delimiter_from_token_stream(
+    token_generator: Iterator[TI],
+    line_lengths: list[int],
+    candidate: int,
+) -> tuple[int, int] | None:
+    brackets: list[tuple[str, int]] = []
+    strings: list[tuple[int, int]] = []
+    closing = {")": "(", "]": "[", "}": "{"}
+    for token in token_generator:
+        if token.type == T.ERRORTOKEN and token.string in {"\"", "'"}:
+            # An unfinished short string can expose apparent operators.
+            break
+        offset = line_lengths[token.start[0] - 1] + token.start[1]
+        pair = None
+        match token.type:
+            case T.STRING | T.FSTRING_START | T.TSTRING_START:
+                # Token boundaries already account for prefixes and escapes.
+                quote_index = next(
+                    i for i, c in enumerate(token.string) if c in "\"'"
+                )
+                delimiter = token.string[quote_index]
+                width = 3 if token.string.startswith(delimiter * 3, quote_index) else 1
+                opening = (offset + quote_index, offset + quote_index + width)
+                if token.type == T.STRING:
+                    end = line_lengths[token.end[0] - 1] + token.end[1]
+                    pair = (opening, (end - width, end))
+                else:
+                    strings.append(opening)
+            case T.FSTRING_END | T.TSTRING_END:
+                opening = strings.pop()
+                pair = (opening, (offset, offset + len(token.string)))
+            case T.OP if token.string in "()[]{}":
+                if token.string in "([{":
+                    brackets.append((token.string, offset))
+                elif brackets and brackets[-1][0] == closing[token.string]:
+                    _, opening_offset = brackets.pop()
+                    pair = (
+                        (opening_offset, opening_offset + 1), (offset, offset + 1)
+                    )
+                else:
+                    # Do not pair across an incorrectly nested delimiter.
+                    brackets.clear()
+        if pair is not None:
+            opening, ending = pair
+            if opening[0] <= candidate < opening[1]:
+                return ending
+            if ending[0] <= candidate < ending[1]:
+                return opening
+    return None
+
+
+def gen_colors(buffer: str) -> Iterator[ColorSpan]:
+    """Returns a list of index spans to color using the given color tag.
+
+    The input `buffer` should be a valid start of a Python code block, i.e.
+    it cannot be a block starting in the middle of a multiline string.
+    """
+    gen, line_lengths = _get_token_stream(buffer)
     last_emitted: ColorSpan | None = None
     try:
         for color in gen_colors_from_token_stream(gen, line_lengths):

Lib/test/test_pyrepl/test_interact.py

@@ -8,9 +8,52 @@
 from test.support import force_not_colorized
 
 from _pyrepl.console import InteractiveColoredConsole
-from _pyrepl.simple_interact import _more_lines
+from _pyrepl.simple_interact import _more_lines, run_multiline_interactive_console
+from .support import FakeConsole, prepare_reader
 
 class TestSimpleInteract(unittest.TestCase):
+    def test_keyboard_interrupt_clears_matching_delimiter_overlay(self):
+        backend = FakeConsole(())
+        reader = prepare_reader(backend)
+        reader.can_colorize = True
+        reader.buffer = list("(x)")
+        reader.pos = 0
+        reader.refresh()
+
+        matching_cells = [
+            cell
+            for overlay in reader.rendered_screen.overlays
+            for line in overlay.lines
+            for cell in line.cells
+            if cell.style.tag == "matching_delimiter"
+        ]
+        self.assertEqual([cell.text for cell in matching_cells], [")"])
+
+        console = InteractiveColoredConsole()
+        reader.push_input_trans(reader.isearch_trans)
+        with (
+            patch("_pyrepl.readline._setup"),
+            patch("_pyrepl.simple_interact._get_reader", return_value=reader),
+            patch(
+                "_pyrepl.simple_interact.multiline_input",
+                side_effect=[KeyboardInterrupt, EOFError],
+            ),
+            patch.object(console, "runsource", wraps=console.runsource) as runsource,
+            contextlib.redirect_stderr(io.StringIO()) as stderr,
+        ):
+            run_multiline_interactive_console(console)
+
+        self.assertEqual("".join(reader.buffer), "(x)")
+        self.assertFalse(
+            any(
+                cell.style.tag == "matching_delimiter"
+                for line in reader.rendered_screen.composed_lines
+                for cell in line.cells
+            )
+        )
+        self.assertIn("KeyboardInterrupt", stderr.getvalue())
+        runsource.assert_not_called()
+
     def test_multiple_statements(self):
         namespace = {}
         code = dedent("""\