diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index edc40952254029f..b0bf527f4f0ec19 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags( PyCompilerFlags *flags); +PyAPI_FUNC(PyObject *) Py_CompileStringFlags( + const char *str, + const char *filename, + int start, + PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( const char *str, const char *filename, /* decoded from the filesystem encoding */ diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index b2126f3f71bb46e..d333eb2ccf7c413 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -28,7 +28,7 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); -extern PyObject * _Py_CompileStringObjectWithModule( +extern PyObject * _Py_CompileString( const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize, diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index d0418cdee628677..14f9833f63a4fb8 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -1,20 +1,27 @@ +import ast import os import sys import tempfile +import textwrap import unittest from collections import UserDict from test import support from test.support import import_helper from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + NULL = None -_testcapi = import_helper.import_module('_testcapi') Py_single_input = _testcapi.Py_single_input Py_file_input = _testcapi.Py_file_input Py_eval_input = _testcapi.Py_eval_input +INVALID_START = Py_single_input - 1 STDIN = '' STDERR_FD = 2 +PyCF_ONLY_AST = _testcapi.PyCF_ONLY_AST +PyCF_IGNORE_COOKIE = _testcapi.PyCF_IGNORE_COOKIE # Code raising a SyntaxError SYNTAX_ERROR = 'True = 1' @@ -68,6 +75,9 @@ def run(s, *args): run(b'raise ValueError("BUG")', {}) self.assertEqual(str(cm.exception), 'BUG') + with self.assertRaises(ValueError): + func(b'x = 1', INVALID_START, {}) + self.assertIsNone(run(b'a\n', dict(a=1))) self.assertIsNone(run(b'a\n', dict(a=1), {})) self.assertIsNone(run(b'a\n', {}, dict(a=1))) @@ -118,6 +128,9 @@ def run(*args): closeit = 1 self.assertIsNone(run(dict(a=1), {}, closeit)) + with self.assertRaises(ValueError): + func(filename, INVALID_START, {}) + self.assertRaises(NameError, run, {}) self.assertRaises(NameError, run, {}, {}) self.assertRaises(TypeError, run, dict(a=1), []) @@ -357,6 +370,65 @@ def test_run_simplestringflags(self): # Test PyRun_SimpleStringFlags() self.check_run_simplestring(_testcapi.run_simplestringflags) + def check_compilestring(self, compilestring, has_flags, encode_filename=True): + filename_str = TESTFN + if encode_filename: + filename = os.fsencode(filename_str) + else: + filename = filename_str + + def check_code(co, name, value): + ns = {} + exec(co, ns, ns) + self.assertEqual(ns[name], value) + + co = compilestring(b'x = 1', filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 1) + + if has_flags: + code = textwrap.dedent(""" + # encoding: latin1 + x = 'a\xe9' + """) + co = compilestring(code.encode(), filename, Py_file_input, PyCF_IGNORE_COOKIE) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xe9') + + co = compilestring(code.encode(), filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xc3\xa9') + + tree = compilestring(b'x = 1', filename, Py_file_input, PyCF_ONLY_AST) + self.assertIsInstance(tree, ast.AST) + + co = compilestring(b'raise ValueError("BUG")', filename, Py_file_input) + with self.assertRaises(ValueError): + exec(co, {}) + + with self.assertRaises(SyntaxError) as cm: + compilestring(SYNTAX_ERROR.encode(), filename, Py_file_input) + + with self.assertRaises(ValueError): + compilestring(b'x = 1', filename, INVALID_START) + + def test_compilestring(self): + # Test Py_CompileString() + self.check_compilestring(_testlimitedcapi.run_compilestring, False) + + def test_compilestringflags(self): + # Test Py_CompileStringFlags() + self.check_compilestring(_testcapi.run_compilestringflags, True) + + def test_compilestringexflags(self): + # Test Py_CompileStringExFlags() + self.check_compilestring(_testcapi.run_compilestringexflags, True) + + def test_compilestringobject(self): + # Test Py_CompileStringObject() + self.check_compilestring(_testcapi.run_compilestringobject, True, + encode_filename=False) + if __name__ == '__main__': unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8efea27824f0e88..524f1466b191f64 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index 7fc180e136559ba..6812af994a43302 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -24,34 +24,16 @@ #undef PyRun_InteractiveLoop -// Some PyRun functions crash if start is invalid, -// so validate the start argument. -static int -check_start(int start) -{ - if (start == Py_single_input || start == Py_file_input - || start == Py_eval_input || start == Py_func_type_input) - { - return 0; - } - PyErr_SetString(PyExc_ValueError, "invalid start argument"); - return -1; -} - - // Test PyRun_String() static PyObject* run_string(PyObject *mod, PyObject *args) { const char *str; - int start = 0; + int start; PyObject *globals = NULL; PyObject *locals = NULL; - if (!PyArg_ParseTuple(args, "y|iOO", &str, &start, &globals, &locals)) { - return NULL; - } - if (check_start(start) < 0) { + if (!PyArg_ParseTuple(args, "yi|OO", &str, &start, &globals, &locals)) { return NULL; } NULLABLE(globals); @@ -79,9 +61,6 @@ run_stringflags(PyObject *mod, PyObject *args) &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -481,9 +460,6 @@ run_file(PyObject *mod, PyObject *args) &globals, &locals)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); @@ -518,9 +494,6 @@ run_fileex(PyObject *mod, PyObject *args) &closeit)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); @@ -560,9 +533,6 @@ run_fileflags(PyObject *mod, PyObject *args) &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -606,9 +576,6 @@ run_fileexflags(PyObject *mod, PyObject *args) &closeit, &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -733,6 +700,86 @@ run_interactiveloopflags(PyObject *mod, PyObject *args) } +// Test Py_CompileStringFlags() +static PyObject* +run_compilestringflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(args, "yyi|ii", &str, &filename, &start, + &cf_flags, &cf_feature_version)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringFlags(str, filename, start, pflags); +} + + +// Test Py_CompileStringExFlags() +static PyObject* +run_compilestringexflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yyi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringExFlags(str, filename, start, pflags, optimize); +} + + +// Test Py_CompileStringObject() +static PyObject* +run_compilestringobject(PyObject *mod, PyObject *args) +{ + const char *str; + PyObject *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yOi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringObject(str, filename, start, pflags, optimize); +} + + static PyMethodDef test_methods[] = { {"run_string", run_string, METH_VARARGS}, {"run_stringflags", run_stringflags, METH_VARARGS}, @@ -754,6 +801,9 @@ static PyMethodDef test_methods[] = { {"run_simplestringflags", run_simplestringflags, METH_VARARGS}, {"run_interactiveloop", run_interactiveloop, METH_VARARGS}, {"run_interactiveloopflags", run_interactiveloopflags, METH_VARARGS}, + {"run_compilestringflags", run_compilestringflags, METH_VARARGS}, + {"run_compilestringexflags", run_compilestringexflags, METH_VARARGS}, + {"run_compilestringobject", run_compilestringobject, METH_VARARGS}, {NULL}, }; @@ -763,5 +813,11 @@ _PyTestCapi_Init_Run(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } + if (PyModule_AddIntMacro(mod, PyCF_ONLY_AST) < 0) { + return -1; + } + if (PyModule_AddIntMacro(mod, PyCF_IGNORE_COOKIE) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 9314fccc6c915a4..8ce704502af0104 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -101,5 +101,8 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Run(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index c51d285e19ab0d6..a11e0edb8311e31 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -46,5 +46,6 @@ int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); int _PyTestLimitedCAPI_Init_Version(PyObject *module); int _PyTestLimitedCAPI_Init_File(PyObject *module); int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); +int _PyTestLimitedCAPI_Init_Run(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/run.c b/Modules/_testlimitedcapi/run.c new file mode 100644 index 000000000000000..ca12e5faa5b3f96 --- /dev/null +++ b/Modules/_testlimitedcapi/run.c @@ -0,0 +1,35 @@ +#include "parts.h" +#include "util.h" + + +// Test functions, not macros +#undef Py_CompileString + + +// Test Py_CompileString() +static PyObject* +run_compilestring(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + + if (!PyArg_ParseTuple(args, "yyi", &str, &filename, &start)) { + return NULL; + } + + return Py_CompileString(str, filename, start); +} + + +static PyMethodDef test_methods[] = { + {"run_compilestring", run_compilestring, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Run(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 69558d204dbb8e7..7ddcee6d9735ce6 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -118,6 +118,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 2bcc3f6ff176bd9..66a0a47d8e5548b 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -34,6 +34,7 @@ + diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fa64255be00e75d..cbe59c8883d5a57 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -960,9 +960,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = _Py_CompileStringObjectWithModule(str, filename, - start[compile_mode], &cf, - optimize, modname); + result = _Py_CompileString(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5abfdc04538c369..049eaea4994ac46 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1240,11 +1240,27 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } +static int +check_start(int start) +{ + if (start == Py_single_input || start == Py_file_input + || start == Py_eval_input || start == Py_func_type_input) + { + return 0; + } + PyErr_SetString(PyExc_ValueError, "invalid start argument"); + return -1; +} + static PyObject * _PyRun_String(const char *str, PyObject* name, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, int generate_new_source) { + if (check_start(start) < 0) { + return NULL; + } + PyObject *ret = NULL; mod_ty mod; PyArena *arena; @@ -1294,6 +1310,10 @@ static PyObject * _PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) { + if (check_start(start) < 0) { + return NULL; + } + PyArena *arena = _PyArena_New(); if (arena == NULL) { return NULL; @@ -1538,14 +1558,17 @@ PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) { - return _Py_CompileStringObjectWithModule(str, filename, start, - flags, optimize, NULL); + return _Py_CompileString(str, filename, start, flags, optimize, NULL); } PyObject * -_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, - PyCompilerFlags *flags, int optimize, PyObject *module) +_Py_CompileString(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { + if (check_start(start) < 0) { + return NULL; + } + PyCodeObject *co; mod_ty mod; PyArena *arena = _PyArena_New(); @@ -1754,7 +1777,7 @@ Py_CompileString(const char *str, const char *p, int s) } #undef Py_CompileStringFlags -PyAPI_FUNC(PyObject *) +PyObject* Py_CompileStringFlags(const char *str, const char *p, int s, PyCompilerFlags *flags) {