GH-137573: Check C stack depth before stack allocating JIT optimizer struct

Lib/test/test_capi/test_opt.py

@@ -10,11 +10,12 @@
 
 from test.support import (script_helper, requires_specialization,
                           import_helper, Py_GIL_DISABLED, requires_jit_enabled,
-                          reset_code)
+                          reset_code, infinite_recursion)
 
 _testinternalcapi = import_helper.import_module("_testinternalcapi")
 
 from _testinternalcapi import TIER2_THRESHOLD
+from _testcapi import pyobject_vectorcall
 
 
 @contextlib.contextmanager
@@ -2502,6 +2503,37 @@ def testfunc(n):
         self.assertIn("_BINARY_OP", uops)
 
 
+@requires_jit_enabled
+class TestStackUse(unittest.TestCase):
+
+    def test_jit_at_depth(self):
+
+        def run_at_depth(n, func):
+            if n:
+                pyobject_vectorcall(run_at_depth, (n-1, func), None)
+            else:
+                func()
+
+        def find_stack_limit():
+            nonlocal depth
+            depth += 1
+            pyobject_vectorcall(find_stack_limit, (), None)
+
+        with infinite_recursion():
+            depth = 0
+            try:
+                find_stack_limit()
+            except RecursionError:
+                pass
+
+            def loop_func():
+                r = 0
+                for i in range(TIER2_THRESHOLD):
+                    r += i
+                return r
+            run_at_depth(depth-1, loop_func)
+
+
 def global_identity(x):
     return x
 

Misc/NEWS.d/next/Core_and_Builtins/2025-08-12-15-02-24.gh-issue-137573.9J6xfX.rst

@@ -0,0 +1 @@
+Guard against C stack overflow when JIT compiling.

Python/optimizer.c

@@ -114,6 +114,12 @@ _PyOptimizer_Optimize(
     _PyInterpreterFrame *frame, _Py_CODEUNIT *start,
     _PyExecutorObject **executor_ptr, int chain_depth)
 {
+    /* Make sure we have enough C stack space for the buffer */
+    int margin = 1 + sizeof(_PyUOpInstruction) * UOP_MAX_TRACE_LENGTH / _PyOS_STACK_MARGIN_BYTES;
+    if (_Py_ReachedRecursionLimitWithMargin(_PyThreadState_GET(), margin)) {
+        return 0;
+    }
+
     _PyStackRef *stack_pointer = frame->stackpointer;
     assert(_PyInterpreterState_GET()->jit);
     // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must*

Python/optimizer_analysis.c

@@ -695,6 +695,11 @@ _Py_uop_analyze_and_optimize(
 )
 {
     OPT_STAT_INC(optimizer_attempts);
+    /* Make sure we have enough C stack space for the optimizer */
+    int margin = 1 + sizeof(JitOptContext)/_PyOS_STACK_MARGIN_BYTES;
+    if (_Py_ReachedRecursionLimitWithMargin(_PyThreadState_GET(), margin)) {
+        return 0;
+    }
 
     int err = remove_globals(frame, buffer, length, dependencies);
     if (err <= 0) {