Skip to content

Commit 3c1848d

Browse files
encukouvstinner
encukou
and
vstinner
authored
gh-157710: Add more asserts to internal PyUnicode operations (GH-159033)
Co-authored-by: Victor Stinner <vstinner@python.org>
1 parent 2ac2b93 commit 3c1848d

4 files changed

Lines changed: 84 additions & 34 deletions

File tree

‎Include/cpython/unicodeobject.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,11 @@ typedef struct {
186186
(assert(PyUnicode_Check(op)), \
187187
_Py_CAST(PyASCIIObject*, (op)))
188188
#define _PyCompactUnicodeObject_CAST(op) \
189-
(assert(PyUnicode_Check(op)), \
189+
(assert(!(_PyASCIIObject_CAST(op)->state.ascii \
190+
&& _PyASCIIObject_CAST(op)->state.compact)), \
190191
_Py_CAST(PyCompactUnicodeObject*, (op)))
191192
#define _PyUnicodeObject_CAST(op) \
192-
(assert(PyUnicode_Check(op)), \
193+
(assert(!_PyASCIIObject_CAST(op)->state.compact), \
193194
_Py_CAST(PyUnicodeObject*, (op)))
194195

195196

‎Include/internal/pycore_unicodeobject.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ extern "C" {
1111
#include "pycore_fileutils.h" // _Py_error_handler
1212
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
1313
#include "pycore_runtime.h" // _Py_LATIN1_CHR()
14+
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE
1415

1516

1617
// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111).
@@ -106,6 +107,21 @@ _PyUnicode_EnsureUnicode(PyObject *obj)
106107
return 0;
107108
}
108109

110+
static inline char*
111+
_PyUnicode_UTF8(PyObject *op)
112+
{
113+
return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8);
114+
}
115+
116+
/* true if the Unicode object has an allocated UTF-8 memory block
117+
(not shared with other data) */
118+
static inline int _PyUnicode_HAS_UTF8_MEMORY(PyObject *op)
119+
{
120+
return (!PyUnicode_IS_COMPACT_ASCII(op)
121+
&& _PyUnicode_UTF8(op) != NULL
122+
&& _PyUnicode_UTF8(op) != PyUnicode_DATA(op));
123+
}
124+
109125
#ifndef NDEBUG
110126
static inline int
111127
_PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
@@ -123,6 +139,7 @@ _PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer)
123139
assert(PyUnstable_Unicode_GET_CACHED_HASH(buffer) == -1);
124140
assert(!PyUnicode_CHECK_INTERNED(buffer));
125141
assert(!_Py_IsImmortal(buffer));
142+
assert(!_PyUnicode_HAS_UTF8_MEMORY(buffer));
126143
return 1;
127144
}
128145
#endif
@@ -439,6 +456,10 @@ extern int _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *);
439456
// Export for test_peg_generator
440457
PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *);
441458

459+
#ifdef Py_DEBUG
460+
PyAPI_FUNC(void) _PyUnicode_Dump(PyObject *op);
461+
#endif
462+
442463
/* --- Runtime lifecycle -------------------------------------------------- */
443464

444465
extern void _PyUnicode_InitState(PyInterpreterState *);

‎Lib/test/test_capi/test_unicode.py‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from test import support
55
from test.support import import_helper
66
from test.support import threading_helper
7-
from test.support.script_helper import assert_python_failure
7+
from test.support.script_helper import assert_python_failure, assert_python_ok
88
from threading import Thread
99

1010
try:
@@ -1966,6 +1966,47 @@ def copy(text):
19661966
# CRASHES unicode_equal("abc", NULL)
19671967
# CRASHES unicode_equal(NULL, "abc")
19681968

1969+
def test_pyunicode_dump(self):
1970+
try:
1971+
ctypes.pythonapi._PyUnicode_Dump
1972+
except AttributeError:
1973+
self.skipTest("_PyUnicode_Dump not available")
1974+
proc = assert_python_ok('-c', """
1975+
import sys, ctypes
1976+
_PyUnicode_Dump = ctypes.pythonapi._PyUnicode_Dump
1977+
_PyUnicode_Dump.argtypes = [ctypes.py_object]
1978+
_PyUnicode_Dump.restype = None
1979+
PyUnicode_AsUTF8 = ctypes.pythonapi.PyUnicode_AsUTF8
1980+
PyUnicode_AsUTF8.argtypes = [ctypes.py_object]
1981+
PyUnicode_AsUTF8.restype = ctypes.c_char_p
1982+
for s in (
1983+
"ASCII parrot", "latin1 møøse",
1984+
"UCS2 half‐a‐bee", "UCS4 \N{RABBIT}"
1985+
):
1986+
print(s, flush=True)
1987+
_PyUnicode_Dump(s)
1988+
PyUnicode_AsUTF8(s)
1989+
_PyUnicode_Dump(s)
1990+
sys.stdout.flush()
1991+
""".encode(), PYTHONIOENCODING='UTF-8')
1992+
stdout = proc.out.decode().replace('\r', '')
1993+
self.assertRegex(stdout, textwrap.dedent(r"""
1994+
\A
1995+
ASCII parrot\n
1996+
ascii: len=12, data=[^\n]*\n
1997+
ascii: len=12, data=[^\n]*\n
1998+
latin1 møøse\n
1999+
latin1: len=12, utf8=NULL \(0\), data=[^\n]*\n
2000+
latin1: len=12, utf8=[^\n]* \(14\), data=[^\n]*\n
2001+
UCS2 half‐a‐bee\n
2002+
UCS2: len=15, utf8=NULL \(0\), data=[^\n]*\n
2003+
UCS2: len=15, utf8=[^\n]* \(19\), data=[^\n]*\n
2004+
UCS4 \N{RABBIT}\n
2005+
UCS4: len=6, utf8=NULL \(0\), data=[^\n]*\n
2006+
UCS4: len=6, utf8=[^\n]* \(9\), data=[^\n]*\n
2007+
\Z
2008+
""").strip().replace('\n', ''))
2009+
19692010

19702011
class PyUnicodeWriterTest(unittest.TestCase):
19712012
def create_writer(self, size):

‎Objects/unicodeobject.c‎

Lines changed: 18 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,6 @@ NOTE: In the interpreter's initialization phase, some globals are currently
114114
# define _PyUnicode_CHECK(op) PyUnicode_Check(op)
115115
#endif
116116

117-
static inline char* _PyUnicode_UTF8(PyObject *op)
118-
{
119-
return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8);
120-
}
121-
122117
static inline char* PyUnicode_UTF8(PyObject *op)
123118
{
124119
assert(_PyUnicode_CHECK(op));
@@ -175,15 +170,6 @@ static inline int _PyUnicode_SHARE_UTF8(PyObject *op)
175170
return (_PyUnicode_UTF8(op) == PyUnicode_DATA(op));
176171
}
177172

178-
/* true if the Unicode object has an allocated UTF-8 memory block
179-
(not shared with other data) */
180-
static inline int _PyUnicode_HAS_UTF8_MEMORY(PyObject *op)
181-
{
182-
return (!PyUnicode_IS_COMPACT_ASCII(op)
183-
&& _PyUnicode_UTF8(op) != NULL
184-
&& _PyUnicode_UTF8(op) != PyUnicode_DATA(op));
185-
}
186-
187173

188174
#define LATIN1 _Py_LATIN1_CHR
189175

@@ -1251,34 +1237,32 @@ const void *_PyUnicode_data(void *unicode_raw) {
12511237
printf("compact %d\n", PyUnicode_IS_COMPACT(unicode));
12521238
printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode));
12531239
printf("ascii op %p\n", (void*)(_PyASCIIObject_CAST(unicode) + 1));
1254-
printf("compact op %p\n", (void*)(_PyCompactUnicodeObject_CAST(unicode) + 1));
1255-
printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode));
1240+
if (!PyUnicode_IS_COMPACT_ASCII(unicode)) {
1241+
printf("compact op %p\n", (void*)(_PyCompactUnicodeObject_CAST(unicode) + 1));
1242+
printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode));
1243+
}
12561244
return PyUnicode_DATA(unicode);
12571245
}
12581246

12591247
void
12601248
_PyUnicode_Dump(PyObject *op)
12611249
{
12621250
PyASCIIObject *ascii = _PyASCIIObject_CAST(op);
1263-
PyCompactUnicodeObject *compact = _PyCompactUnicodeObject_CAST(op);
1264-
PyUnicodeObject *unicode = _PyUnicodeObject_CAST(op);
1265-
const void *data;
12661251

1267-
if (ascii->state.compact)
1268-
{
1269-
if (ascii->state.ascii)
1270-
data = (ascii + 1);
1271-
else
1272-
data = (compact + 1);
1273-
}
1274-
else
1275-
data = unicode->data.any;
1276-
printf("%s: len=%zu, ", unicode_kind_name(op), ascii->length);
1252+
printf("%s: len=%zu", unicode_kind_name(op), ascii->length);
12771253

12781254
if (!ascii->state.ascii) {
1279-
printf("utf8=%p (%zu)", (void *)compact->utf8, compact->utf8_length);
1255+
PyCompactUnicodeObject *compact = _PyCompactUnicodeObject_CAST(op);
1256+
if (compact->utf8 == NULL) {
1257+
printf(", utf8=NULL");
1258+
}
1259+
else {
1260+
printf(", utf8=%p", (void *)compact->utf8);
1261+
}
1262+
printf(" (%zu)", compact->utf8_length);
12801263
}
1281-
printf(", data=%p\n", data);
1264+
printf(", data=%p\n", PyUnicode_DATA(op));
1265+
fflush(stdout);
12821266
}
12831267
#endif
12841268

@@ -1767,6 +1751,9 @@ _PyUnicode_IsModifiable(PyObject *unicode)
17671751
return 0;
17681752
if (PyUnicode_CHECK_INTERNED(unicode))
17691753
return 0;
1754+
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
1755+
return 0;
1756+
}
17701757
#ifdef Py_DEBUG
17711758
/* singleton refcount is greater than 1 */
17721759
assert(!unicode_is_singleton(unicode));

0 commit comments

Comments
 (0)