Retry Bun backtest cells on hosted-fetch 5xx blips

scripts/backtest-bun.py

@@ -187,11 +187,12 @@ def save(path, data):
 BUN_TRANSPORT_FAILURE = re.compile(
     r'^error: (?:Connection\w+|FailedToOpenSocket|Timeout|TLSHandshakeTimeout) downloading '
     r'|^error: GET \S+ - 5\d\d\b', re.M)
-
 # The harness's own fetches (published record, hosted tarball digest) that
 # still fail after their in-place retries surface as the cell's `error`:
-# `<urlopen error [Errno 104] Connection reset by peer>`.
-HARNESS_TRANSPORT_FAILURE = re.compile(r'^<urlopen error |^\[(?:Win)?Errno \d+\] Connection reset')
+# a 5xx / 429 (`HTTP Error 503: Service Unavailable`) or no response at all
+# (`<urlopen error [Errno 104] Connection reset by peer>`).
+HARNESS_TRANSPORT_FAILURE = re.compile(
+    r'\bHTTP Error (?:5\d\d|429)\b|^<urlopen error |^\[(?:Win)?Errno \d+\] Connection reset')
 
 
 def bun_transport_failures(output):
@@ -201,7 +202,8 @@ def bun_transport_failures(output):
 
 def has_transport_failure(value):
     """Only explicit request transport errors (a request error or a patch API
-    5xx, from the CLI or from bun's fetch) qualify for a fresh-cell retry."""
+    5xx, from the CLI, from bun's fetch or from the harness's own fetch)
+    qualify for a fresh-cell retry."""
     if isinstance(value, dict):
         return any(has_transport_failure(item) for item in value.values())
     if isinstance(value, list):
@@ -1155,10 +1157,12 @@ def install(binary, label, flags=(), cache=None):
                         ('warmOrdinary', [], 'cache-ordinary')]:
                     if shape == 'production':
                         flags = [*flags, '--production']
-                    code, _ = install(bun, label, flags, cache=cache)
+                    code, output = install(bun, label, flags, cache=cache)
                     correct, hashes = oracle(project, record, 'after')
                     checks[label + 'PatchedBytes'] = code == 0 and correct
                     row[label + 'Files'] = hashes
+                    # Kept so a fetch failure here qualifies the cell for a retry.
+                    row[label + 'InstallTransport'] = bun_transport_failures(output)
                     installed_lock = lock.read_bytes()
                     if (lockb_origin and label == 'ordinary' and installed_lock != patched_lock
                             and ver(version) >= (1, 2, 23)):

scripts/tests/test_backtest_harnesses.py

@@ -69,6 +69,45 @@ def test_a_patch_api_5xx_is_a_transport_failure(self):
         self.assertTrue(bun.has_transport_failure(['API request failed with status 504: error code: 504']))
         self.assertFalse(bun.has_transport_failure({'error': 'API request failed with status 404: not found'}))
 
+    def test_a_harness_fetch_5xx_is_a_transport_failure(self):
+        # run 37852649519: hosted_lockb_digest's urlopen of patch.socket.dev
+        # raised this on all four bun.lockb legs and none of them retried.
+        self.assertTrue(bun.has_transport_failure({'error': 'HTTP Error 503: Service Unavailable'}))
+        self.assertTrue(bun.has_transport_failure({'error': 'HTTP Error 429: Too Many Requests'}))
+        self.assertTrue(bun.has_transport_failure(
+            {'error': '<urlopen error [Errno 104] Connection reset by peer>'}))
+        self.assertFalse(bun.has_transport_failure({'error': 'HTTP Error 404: Not Found'}))
+
+    def test_a_failed_install_fetch_is_a_transport_failure(self):
+        output = 'bun install v1.0.0\nerror: GET https://patch.socket.dev/x.tgz - 503\n'
+        row = dict(passed=False, checks={'frozenPatchedBytes': False},
+                   frozenInstallTransport=bun.bun_transport_failures(output))
+        self.assertTrue(bun.has_transport_failure(row))
+        row['frozenInstallTransport'] = bun.bun_transport_failures('bun install v1.0.0\n')
+        self.assertFalse(bun.has_transport_failure(row))
+
+    def test_a_harness_fetch_5xx_cell_retries_fresh(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            job = ('0.8.1', 'vendored-then-hosted', 'vendored')
+            case = root / 'captures' / '-'.join(job)
+            calls = []
+
+            def run_case(_job):
+                case.mkdir(parents=True)
+                calls.append(True)
+                row = dict(passed=len(calls) > 1, checks={'frozenPatchedBytes': len(calls) > 1})
+                if len(calls) == 1:
+                    row['error'] = 'HTTP Error 503: Service Unavailable'
+                bun.save(case / 'result.json', row)
+                return row
+
+            with patch.object(bun.time, 'sleep'):
+                row = bun.retry_network_cell(run_case, job, root)
+            self.assertTrue(row['passed'])
+            self.assertEqual(len(calls), 2)
+            self.assertEqual(row['networkRetryAttempts'][0]['failedChecks'], ['frozenPatchedBytes'])
+
     def test_functional_failure_is_never_retried(self):
         with tempfile.TemporaryDirectory() as temp:
             row = dict(passed=False, checks={'frozenPatchedBytes': False}, error='installed bytes differ')