From 3d481c81bbb5d8d86443c1e098b912fcb6b92cf5 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 24 Mar 2025 23:49:31 +0000 Subject: [PATCH 01/41] =?UTF-8?q?`test=5Fconcurrent=5Ffutures.test=5Fmap?= =?UTF-8?q?=5Fexception`:=20assert=20no=20reference=20cycle=20from=20faile?= =?UTF-8?q?d=20future=20captured=20in=20its=20exception=E2=80=99s=20traceb?= =?UTF-8?q?ack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_concurrent_futures/executor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index ff7bd0db0c2199c..d739ae7a526e342 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,3 +1,4 @@ +import gc import itertools import threading import time @@ -80,7 +81,20 @@ def test_map_exception(self): i = self.executor.map(divmod, [5, 5, 5, 5], [2, 3, 0, 5]) self.assertEqual(next(i), (2, 1)) self.assertEqual(next(i), (1, 2)) - self.assertRaises(ZeroDivisionError, next, i) + + error = None + try: + next(i) + except ZeroDivisionError as zero_div_error: + error = zero_div_error + self.assertIsNotNone( + error, + msg="next one should raise a ZeroDivisionError", + ) + + # a failed future must not be captured in its + # future._exception.__traceback__ to avoid a reference cycle + self.assertFalse(gc.get_referrers(error)) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) self.assertRaises(StopIteration, next, i) From f6f08a6159a6e536520594855b5b8a5dad92e127 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 25 Mar 2025 00:33:59 +0000 Subject: [PATCH 02/41] test there is no future in referrers --- Lib/test/test_concurrent_futures/executor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index d739ae7a526e342..a97cc40f48df822 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,7 +94,12 @@ def test_map_exception(self): # a failed future must not be captured in its # future._exception.__traceback__ to avoid a reference cycle - self.assertFalse(gc.get_referrers(error)) + self.assertFalse( + any( + isinstance(referrer, futures.Future) + for referrer in gc.get_referrers(error) + ) + ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) self.assertRaises(StopIteration, next, i) From a7a595ddf14905aaf85cde256ec8b2d2efe7c441 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 25 Mar 2025 01:42:12 +0000 Subject: [PATCH 03/41] switch to assertTrue(all(...)) --- Lib/test/test_concurrent_futures/executor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index a97cc40f48df822..60dba51b179dcfd 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,9 +94,9 @@ def test_map_exception(self): # a failed future must not be captured in its # future._exception.__traceback__ to avoid a reference cycle - self.assertFalse( - any( - isinstance(referrer, futures.Future) + self.assertTrue( + all( + not isinstance(referrer, futures.Future) for referrer in gc.get_referrers(error) ) ) From 984d228e9bfd99625f238a46b72d9bcf425cc445 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 25 Mar 2025 11:12:36 +0000 Subject: [PATCH 04/41] add test msg --- Lib/test/test_concurrent_futures/executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 60dba51b179dcfd..c559a31f5449565 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -98,7 +98,8 @@ def test_map_exception(self): all( not isinstance(referrer, futures.Future) for referrer in gc.get_referrers(error) - ) + ), + msg="none of the referrers should be a Future.", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) From 4cd4820826b7abf6dacdd7246d9b1ec15f505bbc Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 25 Mar 2025 11:14:03 +0000 Subject: [PATCH 05/41] format msgs --- Lib/test/test_concurrent_futures/executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index c559a31f5449565..048654d81b7c5d2 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -99,7 +99,7 @@ def test_map_exception(self): not isinstance(referrer, futures.Future) for referrer in gc.get_referrers(error) ), - msg="none of the referrers should be a Future.", + msg="none of the referrers should be a Future", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) @@ -232,7 +232,7 @@ def test_map_buffersize_when_buffer_is_full(self): self.assertEqual( next(ints), buffersize, - msg="should have fetched only `buffersize` elements from `ints`.", + msg="should have fetched only `buffersize` elements from `ints`", ) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() From b7551c8082b7c3c2af2a0eef1e43851916da0464 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 25 Mar 2025 11:23:58 +0000 Subject: [PATCH 06/41] switch to assert list empty for a better assertion error message --- Lib/test/test_concurrent_futures/executor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 048654d81b7c5d2..868b42512e7f669 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,11 +94,12 @@ def test_map_exception(self): # a failed future must not be captured in its # future._exception.__traceback__ to avoid a reference cycle - self.assertTrue( - all( - not isinstance(referrer, futures.Future) + self.assertFalse( + [ + referrer for referrer in gc.get_referrers(error) - ), + if isinstance(referrer, futures.Future) + ], msg="none of the referrers should be a Future", ) self.assertEqual(next(i), (1, 0)) From ec2b0289eadf373d004a76210b7531891d9d3cf6 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Wed, 26 Mar 2025 22:49:32 +0000 Subject: [PATCH 07/41] edit comment --- Lib/test/test_concurrent_futures/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 868b42512e7f669..717c09fa2dd9ebf 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -92,7 +92,7 @@ def test_map_exception(self): msg="next one should raise a ZeroDivisionError", ) - # a failed future must not be captured in its + # a failed future should not be captured in its # future._exception.__traceback__ to avoid a reference cycle self.assertFalse( [ From 171e083f6d3f4f9b6a2002dc3b3199144f89124a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sun, 30 Mar 2025 13:48:04 +0100 Subject: [PATCH 08/41] assert there are no reference cycles at all --- Lib/test/test_concurrent_futures/executor.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 717c09fa2dd9ebf..02ee9c6f43999d2 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,14 +94,7 @@ def test_map_exception(self): # a failed future should not be captured in its # future._exception.__traceback__ to avoid a reference cycle - self.assertFalse( - [ - referrer - for referrer in gc.get_referrers(error) - if isinstance(referrer, futures.Future) - ], - msg="none of the referrers should be a Future", - ) + self.assertListEqual(gc.get_referrers(error), []) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) self.assertRaises(StopIteration, next, i) From 9710843a5d843c10ddc96b11d89eeaf3039e5969 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Sun, 30 Mar 2025 21:20:37 +0100 Subject: [PATCH 09/41] test that traceback frames should not contain any variables referring to an Exception or a failed Future --- Lib/test/test_concurrent_futures/executor.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 02ee9c6f43999d2..e414d7cb4e5a797 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -92,9 +92,22 @@ def test_map_exception(self): msg="next one should raise a ZeroDivisionError", ) - # a failed future should not be captured in its - # future._exception.__traceback__ to avoid a reference cycle - self.assertListEqual(gc.get_referrers(error), []) + self.assertFalse( + gc.get_referrers(error), + msg="the raised error should not have any referrer", + ) + + tb = error.__traceback__ + while (tb := tb.tb_next): + self.assertFalse( + { + var: val + for var, val in tb.tb_frame.f_locals.items() + if isinstance(val, Exception) + or (isinstance(val, futures.Future) and val.exception()) + }, + msg=f"traceback frames should not contain any variables referring to an Exception or a failed Future", + ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) self.assertRaises(StopIteration, next, i) From 00e7aee198a5c40f20d3b1e426148570d0a73721 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Sun, 30 Mar 2025 21:29:05 +0100 Subject: [PATCH 10/41] interpreter.WorkerContext.run: finally unassign exc_wrapper --- Lib/concurrent/futures/interpreter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index fd3d45144b49a71..df1ece324c2e09a 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -88,7 +88,10 @@ def run(self, task): if exc is None: # The exception must have been not shareable. raise # re-raise - raise exc from wrapper + try: + raise exc from wrapper + finally: + exc = None class BrokenInterpreterPool(_thread.BrokenThreadPool): From c62b24e81c0ff816bc31659cb0fa7f873de00e97 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Sun, 30 Mar 2025 21:31:30 +0100 Subject: [PATCH 11/41] fix msg grammar --- Lib/test/test_concurrent_futures/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index e414d7cb4e5a797..d55bcef682783db 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,7 +94,7 @@ def test_map_exception(self): self.assertFalse( gc.get_referrers(error), - msg="the raised error should not have any referrer", + msg="the raised error should not have any referrers", ) tb = error.__traceback__ From a3046c150d0db3e9c68b1f2ab02b4cbb83ee8940 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Sun, 30 Mar 2025 21:47:00 +0100 Subject: [PATCH 12/41] fix access to exception --- Lib/test/test_concurrent_futures/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index d55bcef682783db..0cc8e5824a0a1a6 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -104,7 +104,7 @@ def test_map_exception(self): var: val for var, val in tb.tb_frame.f_locals.items() if isinstance(val, Exception) - or (isinstance(val, futures.Future) and val.exception()) + or (isinstance(val, futures.Future) and val._exception) }, msg=f"traceback frames should not contain any variables referring to an Exception or a failed Future", ) From 7b1021aedc24d224f1c840ed82493de4127c8488 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Sun, 30 Mar 2025 22:34:57 +0100 Subject: [PATCH 13/41] narrow test: traceback frames should not contain any exception --- Lib/test/test_concurrent_futures/executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 0cc8e5824a0a1a6..07953d2e1595631 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -104,9 +104,8 @@ def test_map_exception(self): var: val for var, val in tb.tb_frame.f_locals.items() if isinstance(val, Exception) - or (isinstance(val, futures.Future) and val._exception) }, - msg=f"traceback frames should not contain any variables referring to an Exception or a failed Future", + msg=f"traceback frames should not contain any exception", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) From e617e82c39ae6ac6ee6665cdd56c98666a3d7168 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 10:53:39 +0100 Subject: [PATCH 14/41] test_map_exception: only search for exception that captures itself in its own traceback --- Lib/test/test_concurrent_futures/executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 07953d2e1595631..899bc562e90a078 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -104,8 +104,9 @@ def test_map_exception(self): var: val for var, val in tb.tb_frame.f_locals.items() if isinstance(val, Exception) + and var in val.__traceback__.tb_frame.f_locals }, - msg=f"traceback frames should not contain any exception", + msg=f"the raised exception's traceback should not contain an exception that captures itself in its own traceback", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) From f2e4dbe04fe715e172e507919853272d41c57988 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 11:42:27 +0100 Subject: [PATCH 15/41] concurrent.futures.process: avoid ref cycle in _sendback_result --- Lib/concurrent/futures/process.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 7f4f225c0ad4fbd..20ad281eaf25d82 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -218,9 +218,13 @@ def _sendback_result(result_queue, work_id, result=None, exception=None, result_queue.put(_ResultItem(work_id, result=result, exception=exception, exit_pid=exit_pid)) except BaseException as e: - exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(work_id, exception=exc, - exit_pid=exit_pid)) + result_queue.put( + _ResultItem( + work_id, + exception=_ExceptionWithTraceback(e, e.__traceback__), + exit_pid=exit_pid + ) + ) def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): From 1245a9a95227f2d92273a2a34d0f6a271c7c3a7b Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 11:43:47 +0100 Subject: [PATCH 16/41] concurrent.futures.process: avoid ref cycle in _process_worker --- Lib/concurrent/futures/process.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 20ad281eaf25d82..6b302763ea5ddcb 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -265,9 +265,12 @@ def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=N try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: - exc = _ExceptionWithTraceback(e, e.__traceback__) - _sendback_result(result_queue, call_item.work_id, exception=exc, - exit_pid=exit_pid) + _sendback_result( + result_queue, + call_item.work_id, + exception=_ExceptionWithTraceback(e, e.__traceback__), + exit_pid=exit_pid, + ) else: _sendback_result(result_queue, call_item.work_id, result=r, exit_pid=exit_pid) From e9845926694994bf62c696a05f1790588079b2db Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 12:27:17 +0100 Subject: [PATCH 17/41] format test msg --- Lib/test/test_concurrent_futures/executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 899bc562e90a078..0bfad3f5f4406d7 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,7 +94,7 @@ def test_map_exception(self): self.assertFalse( gc.get_referrers(error), - msg="the raised error should not have any referrers", + msg="the exception should not have any referrers", ) tb = error.__traceback__ @@ -106,7 +106,7 @@ def test_map_exception(self): if isinstance(val, Exception) and var in val.__traceback__.tb_frame.f_locals }, - msg=f"the raised exception's traceback should not contain an exception that captures itself in its own traceback", + msg=f"the exception's traceback should not contain an exception that captures itself in its own traceback", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) From d3dee343569725f992d00f03c20c025f2749b100 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 12:30:04 +0100 Subject: [PATCH 18/41] refactor ZeroDivisionError assert --- Lib/test/test_concurrent_futures/executor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 0bfad3f5f4406d7..544cd04ad34de69 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -85,11 +85,11 @@ def test_map_exception(self): error = None try: next(i) - except ZeroDivisionError as zero_div_error: - error = zero_div_error - self.assertIsNotNone( - error, - msg="next one should raise a ZeroDivisionError", + except Exception as e: + error = e + self.assertTrue( + isinstance(error, ZeroDivisionError), + msg="next should raise a ZeroDivisionError", ) self.assertFalse( From fc2a3517d6aeac4195f4e96780d5247c228f7c3c Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 12:49:22 +0100 Subject: [PATCH 19/41] Revert "concurrent.futures.process: avoid ref cycle in _process_worker" This reverts commit 09b08197db97ad31e23289e7cbb920e41c8706af. --- Lib/concurrent/futures/process.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 6b302763ea5ddcb..20ad281eaf25d82 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -265,12 +265,9 @@ def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=N try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: - _sendback_result( - result_queue, - call_item.work_id, - exception=_ExceptionWithTraceback(e, e.__traceback__), - exit_pid=exit_pid, - ) + exc = _ExceptionWithTraceback(e, e.__traceback__) + _sendback_result(result_queue, call_item.work_id, exception=exc, + exit_pid=exit_pid) else: _sendback_result(result_queue, call_item.work_id, result=r, exit_pid=exit_pid) From 2e2902093e1de56ca04e536277e86b86d3e2092b Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 31 Mar 2025 12:49:37 +0100 Subject: [PATCH 20/41] Revert "concurrent.futures.process: avoid ref cycle in _sendback_result" This reverts commit ef301226c144bc92c64f3bbc5fb78a8c6177d298. --- Lib/concurrent/futures/process.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 20ad281eaf25d82..7f4f225c0ad4fbd 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -218,13 +218,9 @@ def _sendback_result(result_queue, work_id, result=None, exception=None, result_queue.put(_ResultItem(work_id, result=result, exception=exception, exit_pid=exit_pid)) except BaseException as e: - result_queue.put( - _ResultItem( - work_id, - exception=_ExceptionWithTraceback(e, e.__traceback__), - exit_pid=exit_pid - ) - ) + exc = _ExceptionWithTraceback(e, e.__traceback__) + result_queue.put(_ResultItem(work_id, exception=exc, + exit_pid=exit_pid)) def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): From 51269d90b57e2b310807d681069c071d89686d60 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 1 Apr 2025 00:01:37 +0100 Subject: [PATCH 21/41] skip referrers test for free-threading build on win/linux --- Lib/test/test_concurrent_futures/executor.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 544cd04ad34de69..b0669516707c510 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,10 +1,12 @@ import gc import itertools +import sys import threading import time import weakref from concurrent import futures from operator import add +from sysconfig import get_config_var from test import support from test.support import Py_GIL_DISABLED, warnings_helper @@ -92,10 +94,13 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) - self.assertFalse( - gc.get_referrers(error), - msg="the exception should not have any referrers", - ) + # some referrers may remain for free-threading build on Windows/Linux + is_free_threading = '--disable-gil' in get_config_var("CONFIG_ARGS") + if not is_free_threading or sys.platform not in ("linux", "win32"): + self.assertFalse( + gc.get_referrers(error), + msg="the exception should not have any referrers", + ) tb = error.__traceback__ while (tb := tb.tb_next): From ed0e87f8b09175ede0d3cb2aa464b2b1fc56a116 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 1 Apr 2025 00:16:11 +0100 Subject: [PATCH 22/41] call gc.collect for free-threading on linux/windows --- Lib/test/test_concurrent_futures/executor.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index b0669516707c510..75b4a521fe19f55 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,13 +94,17 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) - # some referrers may remain for free-threading build on Windows/Linux - is_free_threading = '--disable-gil' in get_config_var("CONFIG_ARGS") - if not is_free_threading or sys.platform not in ("linux", "win32"): - self.assertFalse( - gc.get_referrers(error), - msg="the exception should not have any referrers", - ) + # a GC collection may be necessary for free-threading build on Windows/Linux + if ( + '--disable-gil' in get_config_var("CONFIG_ARGS") + and sys.platform in ("linux", "win32") + ): + gc.collect() + + self.assertFalse( + gc.get_referrers(error), + msg="the exception should not have any referrers", + ) tb = error.__traceback__ while (tb := tb.tb_next): From b3f3f4a3b080d62b5e997da91cf1097e91efcccf Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 1 Apr 2025 11:52:20 +0100 Subject: [PATCH 23/41] Revert "call gc.collect for free-threading on linux/windows" This reverts commit cae64747fa2954c8cf868695beb094ebee40032c. --- Lib/test/test_concurrent_futures/executor.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 75b4a521fe19f55..b0669516707c510 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -94,17 +94,13 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) - # a GC collection may be necessary for free-threading build on Windows/Linux - if ( - '--disable-gil' in get_config_var("CONFIG_ARGS") - and sys.platform in ("linux", "win32") - ): - gc.collect() - - self.assertFalse( - gc.get_referrers(error), - msg="the exception should not have any referrers", - ) + # some referrers may remain for free-threading build on Windows/Linux + is_free_threading = '--disable-gil' in get_config_var("CONFIG_ARGS") + if not is_free_threading or sys.platform not in ("linux", "win32"): + self.assertFalse( + gc.get_referrers(error), + msg="the exception should not have any referrers", + ) tb = error.__traceback__ while (tb := tb.tb_next): From e3e75cee9599cfc86bbf738570e3442b9721df08 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Tue, 1 Apr 2025 11:52:30 +0100 Subject: [PATCH 24/41] Revert "skip referrers test for free-threading build on win/linux" This reverts commit fc7a569a71d53722f40b31a61018541847aa976b. --- Lib/test/test_concurrent_futures/executor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index b0669516707c510..544cd04ad34de69 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -1,12 +1,10 @@ import gc import itertools -import sys import threading import time import weakref from concurrent import futures from operator import add -from sysconfig import get_config_var from test import support from test.support import Py_GIL_DISABLED, warnings_helper @@ -94,13 +92,10 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) - # some referrers may remain for free-threading build on Windows/Linux - is_free_threading = '--disable-gil' in get_config_var("CONFIG_ARGS") - if not is_free_threading or sys.platform not in ("linux", "win32"): - self.assertFalse( - gc.get_referrers(error), - msg="the exception should not have any referrers", - ) + self.assertFalse( + gc.get_referrers(error), + msg="the exception should not have any referrers", + ) tb = error.__traceback__ while (tb := tb.tb_next): From 70e22a1c8044adbdaf4d0d42232e52060fb5f08f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 3 Apr 2025 08:40:08 +0100 Subject: [PATCH 25/41] more gc referrers --- Lib/test/test_concurrent_futures/executor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 544cd04ad34de69..c1ddee717e954b5 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -6,7 +6,7 @@ from concurrent import futures from operator import add from test import support -from test.support import Py_GIL_DISABLED, warnings_helper +from test.support import Py_GIL_DISABLED, disable_gc, warnings_helper def mul(x, y): @@ -77,6 +77,7 @@ def test_map(self): list(map(pow, range(10), range(10)))) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @disable_gc() def test_map_exception(self): i = self.executor.map(divmod, [5, 5, 5, 5], [2, 3, 0, 5]) self.assertEqual(next(i), (2, 1)) @@ -92,8 +93,9 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) + # TODO: remove only for debugging self.assertFalse( - gc.get_referrers(error), + [gc.get_referrers(x) for x in gc.get_referrers(error)], msg="the exception should not have any referrers", ) From 12abc0e9bf758dca4cb7fb19844e2db097dcf9fe Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Thu, 3 Apr 2025 09:00:02 +0100 Subject: [PATCH 26/41] even more gc referrers --- Lib/test/test_concurrent_futures/executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index c1ddee717e954b5..bc53a32a75d9eb5 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -95,7 +95,11 @@ def test_map_exception(self): # TODO: remove only for debugging self.assertFalse( - [gc.get_referrers(x) for x in gc.get_referrers(error)], + [ + gc.get_referrers(y) + for x in gc.get_referrers(error) + for y in gc.get_referrers(x) + ], msg="the exception should not have any referrers", ) From cac4aac2bf50ddc3a55c8ffb7ad3c9b988249061 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 4 Apr 2025 07:56:24 +0100 Subject: [PATCH 27/41] delete a ref to work_item --- Lib/concurrent/futures/process.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 7f4f225c0ad4fbd..16f151b411875ee 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -490,15 +490,18 @@ def add_call_item_to_queue(self): else: work_item = self.pending_work_items[work_id] - if work_item.future.set_running_or_notify_cancel(): - self.call_queue.put(_CallItem(work_id, - work_item.fn, - work_item.args, - work_item.kwargs), - block=True) - else: - del self.pending_work_items[work_id] - continue + try: + if work_item.future.set_running_or_notify_cancel(): + self.call_queue.put(_CallItem(work_id, + work_item.fn, + work_item.args, + work_item.kwargs), + block=True) + else: + del self.pending_work_items[work_id] + continue + finally: + del work_item def wait_result_broken_or_wakeup(self): # Wait for a result to be ready in the result_queue while checking From 66b98b49cb223d432ef9a6cb26a55f40d3430bce Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 4 Apr 2025 08:24:08 +0100 Subject: [PATCH 28/41] delete reference to result_item before joining process --- Lib/concurrent/futures/process.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 16f151b411875ee..2bcebb95d5a1c89 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -378,16 +378,17 @@ def run(self): return if result_item is not None: self.process_result_item(result_item) - - process_exited = result_item.exit_pid is not None - if process_exited: - p = self.processes.pop(result_item.exit_pid) - p.join() + exit_pid = result_item.exit_pid # Delete reference to result_item to avoid keeping references # while waiting on new results. del result_item + process_exited = exit_pid is not None + if process_exited: + p = self.processes.pop(exit_pid) + p.join() + if process_exited: with self.shutdown_lock: broken = self._replace_dead_worker() From 68fcecb3c400947c35fe5a2d77724b6d4f59b8b5 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 4 Apr 2025 08:45:02 +0100 Subject: [PATCH 29/41] unpack ResultItem and WorkItem --- Lib/concurrent/futures/process.py | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 2bcebb95d5a1c89..3d9ccceffff5c0c 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -377,12 +377,29 @@ def run(self): self.terminate_broken(cause) return if result_item is not None: - self.process_result_item(result_item) exit_pid = result_item.exit_pid + work_id = result_item.work_id + exception = result_item.exeption + result = result_item.result + del result_item + # Process the received a result_item. This can be either the PID of a + # worker that exited gracefully or a _ResultItem + + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + f = work_item.future + del work_item + if exception: + f.set_exception(exception) + else: + f.set_result(result) + del f - # Delete reference to result_item to avoid keeping references + # Delete reference to exception/result to avoid keeping references # while waiting on new results. - del result_item + del exception, result process_exited = exit_pid is not None if process_exited: @@ -534,19 +551,6 @@ def wait_result_broken_or_wakeup(self): return result_item, is_broken, cause - def process_result_item(self, result_item): - # Process the received a result_item. This can be either the PID of a - # worker that exited gracefully or a _ResultItem - - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(result_item.work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - if result_item.exception is not None: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) - def is_shutting_down(self): # Check whether we should start shutting down the executor. executor = self.executor_reference() From 0be8f498eae72247aacd255213f63315323be77a Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 5 Apr 2025 05:48:10 +0100 Subject: [PATCH 30/41] fix typo --- Lib/concurrent/futures/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 3d9ccceffff5c0c..2d6da275a8f53fa 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -379,7 +379,7 @@ def run(self): if result_item is not None: exit_pid = result_item.exit_pid work_id = result_item.work_id - exception = result_item.exeption + exception = result_item.exception result = result_item.result del result_item # Process the received a result_item. This can be either the PID of a From 6c2406d661e0daa26bbefba14fdb21271041a6d1 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Sat, 5 Apr 2025 09:30:32 +0100 Subject: [PATCH 31/41] bit of a hack? --- Lib/concurrent/futures/process.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 2d6da275a8f53fa..d673a72be5b08dd 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -389,13 +389,12 @@ def run(self): work_item = self.pending_work_items.pop(work_id, None) # work_item can be None if another process terminated (see above) if work_item is not None: - f = work_item.future + f_boxed = [work_item.future] del work_item if exception: - f.set_exception(exception) + f_boxed.pop().set_exception(exception) else: - f.set_result(result) - del f + f_boxed.pop().set_result(result) # Delete reference to exception/result to avoid keeping references # while waiting on new results. From 7037c4aa735bb37f84c3658f6b065dd152fe6691 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 11:52:28 +0100 Subject: [PATCH 32/41] Revert investigations --- Lib/concurrent/futures/process.py | 65 +++++++++----------- Lib/test/test_concurrent_futures/executor.py | 10 +-- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index d673a72be5b08dd..7f4f225c0ad4fbd 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -377,34 +377,17 @@ def run(self): self.terminate_broken(cause) return if result_item is not None: - exit_pid = result_item.exit_pid - work_id = result_item.work_id - exception = result_item.exception - result = result_item.result - del result_item - # Process the received a result_item. This can be either the PID of a - # worker that exited gracefully or a _ResultItem - - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - f_boxed = [work_item.future] - del work_item - if exception: - f_boxed.pop().set_exception(exception) - else: - f_boxed.pop().set_result(result) - - # Delete reference to exception/result to avoid keeping references - # while waiting on new results. - del exception, result + self.process_result_item(result_item) - process_exited = exit_pid is not None + process_exited = result_item.exit_pid is not None if process_exited: - p = self.processes.pop(exit_pid) + p = self.processes.pop(result_item.exit_pid) p.join() + # Delete reference to result_item to avoid keeping references + # while waiting on new results. + del result_item + if process_exited: with self.shutdown_lock: broken = self._replace_dead_worker() @@ -507,18 +490,15 @@ def add_call_item_to_queue(self): else: work_item = self.pending_work_items[work_id] - try: - if work_item.future.set_running_or_notify_cancel(): - self.call_queue.put(_CallItem(work_id, - work_item.fn, - work_item.args, - work_item.kwargs), - block=True) - else: - del self.pending_work_items[work_id] - continue - finally: - del work_item + if work_item.future.set_running_or_notify_cancel(): + self.call_queue.put(_CallItem(work_id, + work_item.fn, + work_item.args, + work_item.kwargs), + block=True) + else: + del self.pending_work_items[work_id] + continue def wait_result_broken_or_wakeup(self): # Wait for a result to be ready in the result_queue while checking @@ -550,6 +530,19 @@ def wait_result_broken_or_wakeup(self): return result_item, is_broken, cause + def process_result_item(self, result_item): + # Process the received a result_item. This can be either the PID of a + # worker that exited gracefully or a _ResultItem + + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception is not None: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) + def is_shutting_down(self): # Check whether we should start shutting down the executor. executor = self.executor_reference() diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index bc53a32a75d9eb5..544cd04ad34de69 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -6,7 +6,7 @@ from concurrent import futures from operator import add from test import support -from test.support import Py_GIL_DISABLED, disable_gc, warnings_helper +from test.support import Py_GIL_DISABLED, warnings_helper def mul(x, y): @@ -77,7 +77,6 @@ def test_map(self): list(map(pow, range(10), range(10)))) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() - @disable_gc() def test_map_exception(self): i = self.executor.map(divmod, [5, 5, 5, 5], [2, 3, 0, 5]) self.assertEqual(next(i), (2, 1)) @@ -93,13 +92,8 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) - # TODO: remove only for debugging self.assertFalse( - [ - gc.get_referrers(y) - for x in gc.get_referrers(error) - for y in gc.get_referrers(x) - ], + gc.get_referrers(error), msg="the exception should not have any referrers", ) From 0568262d5f3457145b82efd16e4d5dd26f99181a Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 12:00:15 +0100 Subject: [PATCH 33/41] pause before checking referrers --- Lib/test/test_concurrent_futures/executor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 544cd04ad34de69..3e56e0199bca7f4 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -92,6 +92,8 @@ def test_map_exception(self): msg="next should raise a ZeroDivisionError", ) + # free-threading builds need this pause on Ubuntu (ARM) and Windows + time.sleep(1) self.assertFalse( gc.get_referrers(error), msg="the exception should not have any referrers", From cfeaa9b29b7dcd79d11abfce1a2cd9f744163781 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 14:23:39 +0100 Subject: [PATCH 34/41] add comment about the avoided ref cycle with exc_wrapper --- Lib/concurrent/futures/interpreter.py | 1 + Modules/_opcode-488016f1.o.tmp | 0 2 files changed, 1 insertion(+) create mode 100644 Modules/_opcode-488016f1.o.tmp diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index df1ece324c2e09a..be26b40628ee434 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -91,6 +91,7 @@ def run(self, task): try: raise exc from wrapper finally: + # avoid a ref cycle where exc is captured in its traceback exc = None diff --git a/Modules/_opcode-488016f1.o.tmp b/Modules/_opcode-488016f1.o.tmp new file mode 100644 index 000000000000000..e69de29bb2d1d64 From 346b4eab00654e4c30277ad90420e2cf5898e873 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 14:27:24 +0100 Subject: [PATCH 35/41] format test --- Lib/test/test_concurrent_futures/executor.py | 11 ++++++----- Modules/_opcode-488016f1.o.tmp | 0 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 Modules/_opcode-488016f1.o.tmp diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 3e56e0199bca7f4..e87be7b879ff5a9 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -82,24 +82,25 @@ def test_map_exception(self): self.assertEqual(next(i), (2, 1)) self.assertEqual(next(i), (1, 2)) - error = None + exception = None try: next(i) except Exception as e: - error = e + exception = e self.assertTrue( - isinstance(error, ZeroDivisionError), + isinstance(exception, ZeroDivisionError), msg="next should raise a ZeroDivisionError", ) # free-threading builds need this pause on Ubuntu (ARM) and Windows time.sleep(1) + self.assertFalse( - gc.get_referrers(error), + gc.get_referrers(exception), msg="the exception should not have any referrers", ) - tb = error.__traceback__ + tb = exception.__traceback__ while (tb := tb.tb_next): self.assertFalse( { diff --git a/Modules/_opcode-488016f1.o.tmp b/Modules/_opcode-488016f1.o.tmp deleted file mode 100644 index e69de29bb2d1d64..000000000000000 From 2a25b9a6ab1a8ffdd671f5b439c0fd34206d5214 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 18:01:48 +0100 Subject: [PATCH 36/41] make the self-captured exceptions check not rely on var name --- Lib/test/test_concurrent_futures/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index e87be7b879ff5a9..58165d1ad4ac2cb 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -107,7 +107,7 @@ def test_map_exception(self): var: val for var, val in tb.tb_frame.f_locals.items() if isinstance(val, Exception) - and var in val.__traceback__.tb_frame.f_locals + and val.__traceback__.tb_frame is tb.tb_frame }, msg=f"the exception's traceback should not contain an exception that captures itself in its own traceback", ) From fd9b9bbab70560663813515df11e368d3e311137 Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 20:02:29 +0100 Subject: [PATCH 37/41] use traceback.walk_tb --- Lib/test/test_concurrent_futures/executor.py | 26 +++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 58165d1ad4ac2cb..57b77bf053adbfd 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -2,9 +2,10 @@ import itertools import threading import time +import traceback import weakref from concurrent import futures -from operator import add +from operator import add, itemgetter from test import support from test.support import Py_GIL_DISABLED, warnings_helper @@ -100,17 +101,18 @@ def test_map_exception(self): msg="the exception should not have any referrers", ) - tb = exception.__traceback__ - while (tb := tb.tb_next): - self.assertFalse( - { - var: val - for var, val in tb.tb_frame.f_locals.items() - if isinstance(val, Exception) - and val.__traceback__.tb_frame is tb.tb_frame - }, - msg=f"the exception's traceback should not contain an exception that captures itself in its own traceback", - ) + frames = map(itemgetter(0), traceback.walk_tb(exception.__traceback__)) + next(frames) + self.assertFalse( + [ + (var, val) + for frame in frames + for var, val in frame.f_locals.items() + if isinstance(val, Exception) + and frame in map(itemgetter(0), traceback.walk_tb(val.__traceback__)) + ], + msg=f"the exception's traceback should not contain an exception that captures itself in its own traceback", + ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) self.assertRaises(StopIteration, next, i) From 682a7199629759b0e65b5c585a5affc2756511ef Mon Sep 17 00:00:00 2001 From: ebonnal Date: Mon, 7 Apr 2025 20:20:48 +0100 Subject: [PATCH 38/41] comment the skipping of the current frame --- Lib/test/test_concurrent_futures/executor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 57b77bf053adbfd..b541a2873894c79 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -102,7 +102,10 @@ def test_map_exception(self): ) frames = map(itemgetter(0), traceback.walk_tb(exception.__traceback__)) + + # skip current frame next(frames) + self.assertFalse( [ (var, val) From fb0c5220be1a78e64b5e57484600d67b02ecf92e Mon Sep 17 00:00:00 2001 From: ebonnal Date: Wed, 23 Apr 2025 13:13:16 +0100 Subject: [PATCH 39/41] test_map_exception: improve readability --- Lib/test/test_concurrent_futures/executor.py | 25 ++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index b541a2873894c79..0af0879e3ff0636 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -5,7 +5,7 @@ import traceback import weakref from concurrent import futures -from operator import add, itemgetter +from operator import add from test import support from test.support import Py_GIL_DISABLED, warnings_helper @@ -85,15 +85,15 @@ def test_map_exception(self): exception = None try: - next(i) + i.__next__() except Exception as e: exception = e self.assertTrue( isinstance(exception, ZeroDivisionError), - msg="next should raise a ZeroDivisionError", + msg="should raise a ZeroDivisionError", ) - # free-threading builds need this pause on Ubuntu (ARM) and Windows + # pause needed for free-threading builds on Ubuntu (ARM) and Windows time.sleep(1) self.assertFalse( @@ -101,20 +101,21 @@ def test_map_exception(self): msg="the exception should not have any referrers", ) - frames = map(itemgetter(0), traceback.walk_tb(exception.__traceback__)) - - # skip current frame - next(frames) - self.assertFalse( [ (var, val) - for frame in frames + # go through the frames of the exception's traceback + for frame, _ in traceback.walk_tb(exception.__traceback__) + # skipping the current frame + if frame is not exception.__traceback__.tb_frame + # go through the locals captured in that frame for var, val in frame.f_locals.items() + # check if one of them is an exception if isinstance(val, Exception) - and frame in map(itemgetter(0), traceback.walk_tb(val.__traceback__)) + # check if it is captured in its own traceback + and frame is val.__traceback__.tb_frame ], - msg=f"the exception's traceback should not contain an exception that captures itself in its own traceback", + msg=f"the exception's traceback should not contain an exception captured in its own traceback", ) self.assertEqual(next(i), (1, 0)) self.assertRaises(StopIteration, next, i) From e0a031d1b9ba3726193bc963c5afcce9c263822f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 21 Oct 2026 03:30:27 +0200 Subject: [PATCH 40/41] Fix CI: break exception ref cycle in _MapResultIterator.__next__ (gh-74028) Hold exc across 'raise exc' keeps the frame in the exception's referrers via its traceback; clear it in a finally like the interpreter.py fix so the strengthened test_map_exception passes for all pool types. Co-authored-by: Muse Code (Meta Muse Spark) --- Lib/concurrent/futures/_base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index e728b8e0a91f744..505723b1e44dc5f 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -723,7 +723,11 @@ def __iter__(self): def __next__(self): value, exc = next(self.gen) if exc is not None: - raise exc + try: + raise exc + finally: + # avoid a ref cycle where exc is captured in its traceback + exc = None return value def close(self): From 6c8f6d6bd8d30ea3bf81b1d0e3d30be9452a2f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 21 Oct 2026 20:10:50 +0200 Subject: [PATCH 41/41] test_map_exception: wait for transient referrers instead of sleep(1) The thread that completed the future can still hold the exception (via the future) when the main thread wakes up. Poll with a deadline instead of sleeping a fixed second so the test stays fast normally and robust under load, especially on free-threaded builds. Co-authored-by: Muse Code (Meta Muse Spark) --- Lib/test/test_concurrent_futures/executor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 0af0879e3ff0636..812db1116b3da3b 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -93,8 +93,11 @@ def test_map_exception(self): msg="should raise a ZeroDivisionError", ) - # pause needed for free-threading builds on Ubuntu (ARM) and Windows - time.sleep(1) + # Wait for the completing thread to release its transient reference + # to the exception; do not keep the referrers list alive in the loop. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False): + if not gc.get_referrers(exception): + break self.assertFalse( gc.get_referrers(exception),