gh-158660: Add PyListWriter C API

Doc/c-api/list.rst

@@ -227,3 +227,118 @@ List Objects
 
    Return a new tuple object containing the contents of *list*; equivalent to
    ``tuple(list)``.
+
+
+.. _pylistwriter:
+
+PyListWriter
+------------
+
+The :c:type:`PyListWriter` API can be used to create a Python :class:`list`
+object.
+
+On Free Threading, it's more efficient than the ``PyList`` C API to create a
+:class:`list`, because it can avoid the critical section on the created list.
+
+.. versionadded:: next
+
+
+.. c:type:: PyListWriter
+
+   A list writer object.
+
+   The API is **not thread safe**. To share a writer with multiple threads, a
+   critical section or a lock is needed.
+
+   The instance must be destroyed by :c:func:`PyListWriter_Finish` on
+   success, or :c:func:`PyListWriter_Discard` on error.
+
+
+.. c:function:: PyListWriter* PyListWriter_Create(Py_ssize_t size)
+
+   Create a :c:type:`PyListWriter` and preallocate *size* items.
+
+   On error, set an exception and return ``NULL``.
+
+   *size* must be positive or zero.
+
+.. c:function:: PyObject* PyListWriter_Finish(PyListWriter *writer)
+
+   Finish a :c:type:`PyListWriter` created by
+   :c:func:`PyListWriter_Create`.
+
+   On success, return a Python :class:`list` object.
+   On error, set an exception and return ``NULL``.
+
+   The writer instance is invalid after the call in any case.
+   No API can be called on the writer after :c:func:`PyListWriter_Finish`.
+
+.. c:function:: void PyListWriter_Discard(PyListWriter *writer)
+
+   Discard a :c:type:`PyListWriter` created by :c:func:`PyListWriter_Create`.
+
+   Do nothing if *writer* is ``NULL``.
+
+   The writer instance is invalid after the call.
+   No API can be called on the writer after :c:func:`PyListWriter_Discard`.
+
+.. c:function:: int PyListWriter_Append(PyListWriter *writer, PyObject *item)
+
+   Append an item to the list writer.
+   Similar to :c:func:`PyList_Append`.
+
+   On success, return ``0``.
+   On error, set an exception, leave the writer unchanged, and return ``-1``.
+
+.. c:function:: int PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item)
+
+   Similar to :c:func:`PyListWriter_Append`, but :term:`steals <steal>` the
+   reference to *item*, instead of creating a new :term:`strong reference`.
+
+.. c:function:: int PyListWriter_Extend(PyListWriter *writer, PyObject *iterable)
+
+   Extend the list writer with the contents of *iterable*.
+   Similar to :c:func:`PyList_Extend`.
+
+   On success, return ``0``.
+   On error, set an exception, leave the writer unchanged, and return ``-1``.
+
+   If consuming *iterable* raises an exception, the writer is restored to its
+   previous state: discard added items.
+
+.. c:function:: int PyListWriter_ExtendArray(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem)
+
+   Extend the list writer with the contents of *items*: copy references from
+   *items* to the list writer.
+
+   On success, return ``0``.
+   On error, set an exception, leave the writer unchanged, and return ``-1``.
+
+.. c:function:: int PyListWriter_ExtendArraySteal(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem)
+
+   Similar to :c:func:`PyListWriter_ExtendArray`, but :term:`steal` references
+   instead of copying references.
+
+   On error, all *items* references are also cleared.
+
+.. c:function:: int PyListWriter_FromFormat(PyListWriter *writer, const char *format, ...)
+
+   Add values using :ref:`building values format <arg-parsing>`.
+
+   Example adding 3 integers: ``PyListWriter_FromFormat(writer, "iii", 1, 2, 3)``.
+
+   On success, return ``0``.
+   On error, set an exception, leave the writer unchanged, and return ``-1``.
+
+   On error, the writer is restored to its previous state: discard added items.
+
+.. c:function:: int PyListWriter_FromBuffer(PyListWriter *writer, Py_buffer *view)
+
+   Add values from a buffer.
+
+   *view* must have 1 dimension.
+
+   On success, return ``0``.
+   On error, set an exception, leave the writer unchanged, and return ``-1``.
+
+   On error, the writer is restored to its previous state: discard added items.

Doc/whatsnew/3.16.rst

@@ -1310,6 +1310,24 @@ New features
 * Add :c:func:`Py_HashBuffer` to the limited C API.
   (Contributed by Victor Stinner in :gh:`148233`.)
 
+* Add a new :ref:`PyListWriter <pylistwriter>` C API to create a :class:`list`
+  object. On Free Threading, it's more efficient than the ``PyList`` C API to
+  create a :class:`list`, because it can avoid the critical section on the
+  created list. Add functions:
+
+  * :c:func:`PyListWriter_Create`
+  * :c:func:`PyListWriter_Finish`
+  * :c:func:`PyListWriter_Discard`
+  * :c:func:`PyListWriter_Append`
+  * :c:func:`PyListWriter_AppendSteal`
+  * :c:func:`PyListWriter_Extend`
+  * :c:func:`PyListWriter_ExtendArray`
+  * :c:func:`PyListWriter_ExtendArraySteal`
+  * :c:func:`PyListWriter_FromFormat`
+  * :c:func:`PyListWriter_FromBuffer`
+
+  (Contributed by Victor Stinner in :gh:`158660`.)
+
 * Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C
   API.
   (Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.)

Include/cpython/listobject.h

@@ -51,3 +51,36 @@ PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
 
 PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable);
 PyAPI_FUNC(int) PyList_Clear(PyObject *self);
+
+
+/* --- PyListWriter API --------------------------------------------------- */
+
+typedef struct PyListWriter PyListWriter;
+
+PyAPI_FUNC(PyListWriter*) PyListWriter_Create(Py_ssize_t size);
+PyAPI_FUNC(PyObject*) PyListWriter_Finish(PyListWriter *writer);
+PyAPI_FUNC(void) PyListWriter_Discard(PyListWriter *writer);
+PyAPI_FUNC(int) PyListWriter_Append(
+    PyListWriter *writer,
+    PyObject *item);
+PyAPI_FUNC(int) PyListWriter_AppendSteal(
+    PyListWriter *writer,
+    PyObject *item);
+PyAPI_FUNC(int) PyListWriter_Extend(
+    PyListWriter *writer,
+    PyObject *iterable);
+PyAPI_FUNC(int) PyListWriter_ExtendArray(
+    PyListWriter *writer,
+    PyObject *const *items,
+    Py_ssize_t nitem);
+PyAPI_FUNC(int) PyListWriter_ExtendArraySteal(
+    PyListWriter *writer,
+    PyObject *const *items,
+    Py_ssize_t nitem);
+PyAPI_FUNC(int) PyListWriter_FromFormat(
+    PyListWriter *writer,
+    const char *format,
+    ...);
+PyAPI_FUNC(int) PyListWriter_FromBuffer(
+    PyListWriter *writer,
+    const Py_buffer *buf);

Include/internal/pycore_list.h

@@ -12,6 +12,8 @@ extern "C" {
 #include "pycore_stackref.h"
 #endif
 
+#define _Py_LIST_SMALL_ALLOCATED 32
+
 PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
 PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*);
 PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *);
@@ -77,6 +79,37 @@ union _PyStackRef;
 PyAPI_FUNC(PyObject *)_PyList_FromStackRefStealOnSuccess(const union _PyStackRef *src, Py_ssize_t n);
 PyAPI_FUNC(PyObject *)_PyList_AsTupleAndClear(PyListObject *v);
 
+#ifdef Py_DEBUG
+#  define _PyListWriter_INIT_ITEMS 1
+#else
+#  define _PyListWriter_INIT_ITEMS 0
+#endif
+
+extern int _PyListWriter_Resize(
+    PyListWriter *writer,
+    Py_ssize_t new_size);
+extern void _PyListWriter_ResetSize(
+    PyListWriter *writer,
+    Py_ssize_t size);
+
+
+// The caller must resize the list so it's big enough to set this item.
+// A writer list item must be set exactly once.
+static inline void
+_PyListWriter_SET_ITEM_STEAL(PyListWriter *writer, Py_ssize_t index,
+                             PyObject *item)
+{
+    assert(item != NULL);
+
+    PyListObject *list = (PyListObject*)writer;
+    assert(0 <= index && index < Py_SIZE(list));
+#if _PyListWriter_INIT_ITEMS
+    assert(list->ob_item[index] == NULL);
+#endif
+
+    list->ob_item[index] = item;
+}
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_modsupport.h

@@ -32,12 +32,13 @@ PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t,
     (((min) <= (nargs) && (nargs) <= (max)) \
      || _PyArg_CheckPositional((funcname), (nargs), (min), (max)))
 
-extern PyObject ** _Py_VaBuildStack(
+extern int _Py_VaBuildArray(
+    PyObject ***array,
+    Py_ssize_t *size,
     PyObject **small_stack,
     Py_ssize_t small_stack_len,
     const char *format,
-    va_list va,
-    Py_ssize_t *p_nargs);
+    va_list va);
 
 extern PyObject* _PyModule_CreateInitialized(PyModuleDef*, int apiver);