[3.15] gh-69919: Catch all compile errors in the code module, pyrepl and IDLE (GH-157585) (#158025)

gh-69919: Catch all compile errors in the code module, pyrepl and IDLE (GH-157585)

compile() can raise MemoryError or RecursionError for too deeply
nested source, not only SyntaxError, OverflowError and ValueError.
IDLE's Shell then lost its prompt until the input was deleted.
(cherry picked from commit 4bc392c)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: 4 people

Doc/builtins/functions.rst

@@ -344,8 +344,14 @@ are always available.  They are listed here in alphabetical order.
    It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
    by module name.
 
-   This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled
-   source is invalid.
+   This function raises :exc:`SyntaxError` if the compiled source is invalid,
+   including a *source* containing a null character or that cannot be decoded;
+   :exc:`ValueError` if *mode* or *flags* is invalid,
+   or if a string *source* contains surrogate characters;
+   :exc:`MemoryError` or :exc:`RecursionError` if *source* is too complex
+   to parse or compile,
+   for example an expression with many thousands of nested operators;
+   and :exc:`OverflowError` if *source* is too large.
 
    If you want to parse Python code into its AST representation, see
    :func:`ast.parse`.
@@ -377,11 +383,15 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
-   .. versionadded:: 3.8
+   .. versionchanged:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
       support for top-level ``await``, ``async for``, and ``async with``.
 
-   .. versionadded:: 3.15
+   .. versionchanged:: 3.12
+      :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes
+      are encountered in *source*.
+
+   .. versionchanged:: 3.15
       Added the *module* parameter.
 
 

Doc/library/code.rst

@@ -92,7 +92,7 @@ Interactive Interpreter Objects
    *symbol* is ``'single'``.  One of several things can happen:
 
    * The input is incorrect; :func:`compile_command` raised an exception
-     (:exc:`SyntaxError` or :exc:`OverflowError`).  A syntax traceback will be
+     (usually :exc:`SyntaxError`).  A syntax traceback will be
      printed by calling the :meth:`showsyntaxerror` method.  :meth:`runsource`
      returns ``False``.
 

Lib/_pyrepl/console.py

@@ -261,7 +261,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                 )
             self.showsyntaxerror(filename, source=source)
             return False
-        except (OverflowError, ValueError):
+        except Exception:
             self.showsyntaxerror(filename, source=source)
             return False
         if tree.body:
@@ -282,7 +282,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                     )
                 self.showsyntaxerror(filename, source=source)
                 return False
-            except (OverflowError, ValueError):
+            except Exception:
                 self.showsyntaxerror(filename, source=source)
                 return False
 

Lib/_pyrepl/simple_interact.py

@@ -83,7 +83,7 @@ def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
     src = _strip_final_indent(unicodetext)
     try:
         code = console.compile(src, "<stdin>", "single")
-    except (OverflowError, SyntaxError, ValueError):
+    except Exception:
         lines = src.splitlines(keepends=True)
         if len(lines) == 1:
             return False

Lib/code.py

@@ -44,8 +44,8 @@ def runsource(self, source, filename="<input>", symbol="single"):
         One of several things can happen:
 
         1) The input is incorrect; compile_command() raised an
-        exception (SyntaxError or OverflowError).  A syntax traceback
-        will be printed by calling the showsyntaxerror() method.
+        exception (usually SyntaxError).  A syntax traceback will be
+        printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
@@ -62,7 +62,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
         """
         try:
             code = self.compile(source, filename, symbol)
-        except (OverflowError, SyntaxError, ValueError):
+        except Exception:
             # Case 1
             self.showsyntaxerror(filename, source=source)
             return False

Lib/idlelib/idle_test/test_runscript.py

@@ -29,6 +29,19 @@ def test_init(self):
         sb = runscript.ScriptBinding(ew)
         ew._close()
 
+    def test_checksyntax_compile_error(self):
+        # gh-69919: any error raised by compile() is reported.
+        ew = EditorWindow(root=self.root)
+        sb = runscript.ScriptBinding(ew)
+        sb.flist = mock.Mock()
+        sb.errorbox = mock.Mock()
+        with (mock.patch('idlelib.runscript.compile', create=True,
+                         side_effect=MemoryError()),
+              mock.patch('idlelib.runscript.open', mock.mock_open(read_data=b'x\n'))):
+            self.assertFalse(sb.checksyntax('test.py'))
+        sb.errorbox.assert_called_once_with('MemoryError', '<no detail available>')
+        ew._close()
+
     def test_run_module_event_shell_busy_no_restart(self):
         # gh-82183: running without restarting the busy shell aborts.
         ew = EditorWindow(root=self.root)

Lib/idlelib/pyshell.py

@@ -686,7 +686,7 @@ def execfile(self, filename, source=None):
                               + source + "\ndel __file__")
         try:
             code = compile(source, filename, "exec")
-        except (OverflowError, SyntaxError):
+        except Exception:
             self.tkconsole.resetoutput()
             print('*** Error in script or command!\n'
                  'Traceback (most recent call last):',
@@ -737,19 +737,23 @@ def showsyntaxerror(self, filename=None, **kwargs):
         tkconsole = self.tkconsole
         text = tkconsole.text
         type, value, tb = sys.exc_info()
-        msg = getattr(value, 'msg', '') or value or "<no detail available>"
-        lineno = getattr(value, 'lineno', '') or 1
-        offset = getattr(value, 'offset', '') or 0
+        if not issubclass(type, SyntaxError):
+            tkconsole.resetoutput()
+            InteractiveInterpreter.showsyntaxerror(self, filename, **kwargs)
+            tkconsole.showprompt()
+            return
+        msg = value.msg or "<no detail available>"
+        lineno = value.lineno or 1
+        offset = value.offset or 0
         if offset == 0:
             lineno += 1 #mark end of offending line
         if lineno == 1:
-            pos = "iomark + %d chars" % (offset-1)
+            pos = f"iomark + {offset-1} chars"
         else:
-            pos = "iomark linestart + %d lines + %d chars" % \
-                  (lineno-1, offset-1)
+            pos = f"iomark linestart + {lineno-1} lines + {offset-1} chars"
         tkconsole.colorize_syntax_error(text, pos)
         tkconsole.resetoutput()
-        self.write("SyntaxError: %s\n" % msg)
+        self.write(f"{type.__name__}: {msg}\n")
         tkconsole.showprompt()
 
     def showtraceback(self):

Lib/idlelib/runscript.py

@@ -93,15 +93,19 @@ def checksyntax(self, filename):
         try:
             # If successful, return the compiled code
             return compile(source, filename, "exec")
-        except (SyntaxError, OverflowError, ValueError) as value:
-            msg = getattr(value, 'msg', '') or value or "<no detail available>"
-            lineno = getattr(value, 'lineno', '') or 1
-            offset = getattr(value, 'offset', '') or 0
+        except SyntaxError as value:
+            msg = value.msg or "<no detail available>"
+            lineno = value.lineno or 1
+            offset = value.offset or 0
             if offset == 0:
                 lineno += 1  #mark end of offending line
             pos = "0.0 + %d lines + %d chars" % (lineno-1, offset-1)
             editwin.colorize_syntax_error(text, pos)
-            self.errorbox("SyntaxError", "%-20s" % msg)
+            self.errorbox(type(value).__name__, msg)
+            return False
+        except Exception as value:
+            msg = str(value) or "<no detail available>"
+            self.errorbox(type(value).__name__, msg)
             return False
         finally:
             shell.set_warning_stream(saved_stream)

Lib/pdb.py

@@ -155,7 +155,7 @@ def find_function(funcname, filename):
             if funcdef:
                 try:
                     code = compile(funcdef, filename, 'exec')
-                except SyntaxError:
+                except Exception:
                     continue
                 # We should always be able to find the code object here
                 funccode = next(c for c in code.co_consts if
@@ -2776,7 +2776,7 @@ def _compile_error_message(self, expr):
         """Return the error message as string if compiling `expr` fails."""
         try:
             compile(expr, "<stdin>", "eval")
-        except SyntaxError as exc:
+        except Exception as exc:
             return _rstr(self._format_exc(exc))
         return ""
 

Lib/test/test_code_module.py

@@ -140,6 +140,17 @@ def test_unicode_error(self):
         self.assertIsNone(self.sysmod.last_value.__traceback__)
         self.assertIs(self.sysmod.last_exc, self.sysmod.last_value)
 
+    def test_compile_error(self):
+        # Any error raised by compile() must be reported (gh-69919).
+        self.infunc.side_effect = ['-' * 100_000 + '1', EOFError('Finished')]
+        self.console.interact()
+        output = ''.join(''.join(call[1]) for call in self.stderr.method_calls)
+        output = output[output.index('(InteractiveConsole)'):]
+        output = output[output.index('\n') + 1:]
+        self.assertRegex(output, r'^(MemoryError|RecursionError): ')
+        self.assertIn(self.sysmod.last_type, (MemoryError, RecursionError))
+        self.assertIs(self.sysmod.last_exc, self.sysmod.last_value)
+
     def test_sysexcepthook(self):
         self.infunc.side_effect = ["def f():",
                                    "    raise ValueError('BOOM!')",