From fe2ab7a6a4985ddd8b30b28e14324deb57e6bc17 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:33:38 -0500 Subject: [PATCH] Builds with many modules, or from a long path, no longer pass the shell's 128 KiB limit Patch 0030 hands makeqstrdefs.py its flags and sources in a file instead of on one command line. The Pico 2 build with --modules all named 879 sources by absolute path: 127,095 bytes from a 65-character clone path, 173,951 from a 118-character one, and make stopped with "/bin/sh: Argument list too long". The command is now under 600 bytes whatever the modules or the path. The make ports (unix, windows, webassembly) do the same. Also: the guide names binutils-mingw-w64-x86-64, which has windres; the README says how to report on builds you already have; and build_mp.py --help ends with examples. --- README.md | 6 +- build_mp.py | 15 +- docs/build-plan.md | 2 +- docs/newcomers.md | 14 +- ...es-qstr-arguments-in-a-response-file.patch | 135 ++++++++++++++++++ provenance.json | 3 + 6 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 patches/micropython/0030-micropython-py-mkrules-qstr-arguments-in-a-response-file.patch diff --git a/README.md b/README.md index 6ec3e8f4..aa424d5e 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ New here? The [newcomer's guide](docs/newcomers.md) explains the model; the - [`ppa/`](modules/ppa/): scale, rotate, convert and fill pixel buffers, on the ESP32-P4's Pixel Processing Accelerator or in portable C elsewhere. - [`tsmux/`](modules/tsmux/): MPEG-TS and HLS segments from H.264. -- [`patches/micropython/`](patches/micropython/) — our patch series (`0001-…` to `0029-…`; there +- [`patches/micropython/`](patches/micropython/) — our patch series (`0001-…` to `0030-…`; there is no `0026`, whose fix moved into `patches/tinyusb/`), each justified in its own header; [`patches/apply_patches.py`](patches/apply_patches.py) applies it, plus each module's own MicroPython patches, once, as one local commit. @@ -91,7 +91,9 @@ New here? The [newcomer's guide](docs/newcomers.md) explains the model; the `ESP32_GENERIC_P4` `WIFI6_DEV_KIT` and `WIFI6_DEV_KIT_H2`). - [`scripts/exercise_build.sh`](scripts/exercise_build.sh) — builds every desktop port and two esp32 boards from a bare clone, checks the failure paths, and writes a report of - what should match on any machine. Needs no board. + what should match on any machine. Needs no board. For a report on builds you + already have, name their folders: + `scripts/exercise_build.sh --report builds/rp2/RPI_PICO2/default ...`. - [`provenance.json`](provenance.json) — the patches' checksums and migration record. What's planned next is in [ROADMAP.md](ROADMAP.md). diff --git a/build_mp.py b/build_mp.py index 4b972b4a..bde5ac78 100755 --- a/build_mp.py +++ b/build_mp.py @@ -997,8 +997,21 @@ def failure_hint(port, log_text): return None +EXAMPLES = """examples: + ./build_mp.py asks for port, board, variant and modules + ./build_mp.py --port unix --variant pydevices --modules all + ./build_mp.py --port windows --variant pydevices --modules all + ./build_mp.py --port esp32 --board ESP32_GENERIC_S3 --variant SPIRAM_OCT \\ + --flash 8MB --modules all + ./build_mp.py --port rp2 --board RPI_PICO2 --modules displayif,pygraphics + +Output lands in builds//[/]/. Arguments it doesn't +recognise go to make (-j4, V=1). The guide: docs/newcomers.md.""" + + def main(): - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False) + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False, + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=EXAMPLES) ap.add_argument("--interpreter", choices=("micropython", "circuitpython"), default="micropython", help="circuitpython: CircuitPython-compatible firmware, from CircuitPython's own ports and boards") ap.add_argument("--port") diff --git a/docs/build-plan.md b/docs/build-plan.md index 0f8b4f2e..44cb0d14 100644 --- a/docs/build-plan.md +++ b/docs/build-plan.md @@ -45,7 +45,7 @@ micropython-pydevices/ │ └── jpegio later: see "Open" ├── patches/ │ ├── apply_patches.py the overlay and the modules' own patches, applied once as one local commit -│ ├── micropython/ 0001-…patch to 0029-…patch (no 0026) +│ ├── micropython/ 0001-…patch to 0030-…patch (no 0026) │ └── tinyusb/ patches to the TinyUSB the esp32 port builds (S2, S3, P4) ├── variants/ only where a board truly differs from upstream │ ├── unix/ windows/ pydevices/, vst3-engine/ (as today) diff --git a/docs/newcomers.md b/docs/newcomers.md index 9108fa4d..27a5af26 100644 --- a/docs/newcomers.md +++ b/docs/newcomers.md @@ -46,7 +46,7 @@ fetch are your distribution's packages. On Debian or Ubuntu (WSL included): | unix | `pkg-config libsdl2-dev libffi-dev` (the desktop display) | | esp32 | `ninja-build python3-venv` | | rp2 | `gcc-arm-none-eabi libnewlib-arm-none-eabi` | -| windows, cross-built from Linux | `gcc-mingw-w64-x86-64 libtool libltdl-dev autoconf automake` | +| windows, cross-built from Linux | `gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 libtool libltdl-dev autoconf automake` (the binutils package has `windres`) | | webassembly | `nodejs` | All of it in one line: @@ -54,14 +54,16 @@ All of it in one line: ```bash sudo apt-get update && sudo apt-get install -y git make cmake ninja-build \ python3 python3-venv python3-pip gcc g++ pkg-config libffi-dev libsdl2-dev \ - gcc-mingw-w64-x86-64 libtool libltdl-dev autoconf automake \ + gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64 libtool libltdl-dev \ + autoconf automake \ gcc-arm-none-eabi libnewlib-arm-none-eabi nodejs ``` -The installer prints this line cut down to what your machine is missing, -and says which ports build now. Other distributions carry the same tools -under their own names; a build for a port that's still missing something -stops and says so. +The installer prints only what your machine is missing, one `apt-get install` +line per port and marked with it, so you can leave out the ports you won't +build. It also says which ports build now. Other distributions carry the same +tools under their own names; a build for a port that's still missing +something stops and says so. ### Warnings you can ignore diff --git a/patches/micropython/0030-micropython-py-mkrules-qstr-arguments-in-a-response-file.patch b/patches/micropython/0030-micropython-py-mkrules-qstr-arguments-in-a-response-file.patch new file mode 100644 index 00000000..27fffebe --- /dev/null +++ b/patches/micropython/0030-micropython-py-mkrules-qstr-arguments-in-a-response-file.patch @@ -0,0 +1,135 @@ +From f66b2051c53c936c114f6e0c2abebcae0cc106df Mon Sep 17 00:00:00 2001 +From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> +Date: Mon, 12 Oct 2026 21:30:46 -0500 +Subject: [PATCH] py/mkrules: pass makeqstrdefs.py its arguments in a response + file. + +Make runs each command as one argument to /bin/sh, and Linux caps a +single argument at 128 KiB (MAX_ARG_STRLEN). The qstr preprocessing +command named every QSTR source by its absolute path, and the +preprocessor flags twice, so it grew with every user C module and with +every character of the tree's path. An rp2 RPI_PICO2 build with +micropython-pydevices' --modules all names 879 sources: from a tree at +a 65-character path its command was 127,095 bytes, and from one at a +118-character path 173,951, and make failed before linking: + + make[3]: /bin/sh: Argument list too long + make[3]: *** [.../build.make:987: genhdr/qstr.i.last] Error 127 + +py/mkrules.cmake now writes the flags and sources to genhdr/qstr.i.args +with file(GENERATE), one argument per line, and passes @; the +rule depends on the file, which is rewritten only when its contents +change. py/mkrules.mk does the same for the source lists wherever GNU +make has $(file) (4.0 and later), not only on Windows, and writes one +word per line. makeqstrdefs.py reads a response file as one argument per +line, so a flag with a space or a quote in it arrives intact. The +command's length no longer depends on the sources or the path. make +3.81 (macOS) keeps the command line as it was. + +The preprocessor flags stay on the command line of the step that +preprocesses the collected qstrs: they are passed once there, and gcc +already caps them at 128 KiB in the options it hands cc1. + +Seen building ports/rp2 RPI_PICO2 with micropython-pydevices' modules +from a clone at a 118-character path. Unchanged on upstream v1.29.0. +--- + py/makeqstrdefs.py | 8 +++++--- + py/mkrules.cmake | 18 +++++++++++++++++- + py/mkrules.mk | 22 +++++++++++++++------- + 3 files changed, 37 insertions(+), 11 deletions(-) + +diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py +index f34eaaa3e..7056e2949 100644 +--- a/py/makeqstrdefs.py ++++ b/py/makeqstrdefs.py +@@ -222,13 +222,15 @@ if __name__ == "__main__": + ] + } + +- # An @file argument is a response file: its words stand in its place +- # (py/mkrules.mk passes the source lists that way on Windows). ++ # An @file argument is a response file holding one argument per line, ++ # which stand in its place (py/mkrules.cmake passes everything that ++ # way, and py/mkrules.mk the source lists on Windows). A line is one ++ # argument, so a flag with a space or a quote in it arrives intact. + argv = [] + for arg in sys.argv[1:]: + if arg.startswith("@"): + with open(arg[1:]) as f: +- argv.extend(f.read().split()) ++ argv.extend(line for line in f.read().splitlines() if line) + else: + argv.append(arg) + +diff --git a/py/mkrules.cmake b/py/mkrules.cmake +index 8929db70f..495339364 100644 +--- a/py/mkrules.cmake ++++ b/py/mkrules.cmake +@@ -146,14 +146,30 @@ add_custom_target( + + # Generate qstrs + ++# makeqstrdefs.py takes its flags and sources from a file, one argument per ++# line, instead of from the command line. Make runs a rule's command as one ++# argument to /bin/sh, which Linux caps at 128 KiB (MAX_ARG_STRLEN), and this ++# command named every QSTR source by its absolute path and the flags twice, so ++# it grew with each user C module and with the length of the tree's path. The ++# file is rewritten only when its contents change, and a change reruns the rule. ++set(MICROPY_QSTR_ARGS_FILE ${MICROPY_GENHDR_DIR}/qstr.i.args) ++set(_qstr_args cflags ${MICROPY_CPP_FLAGS} -DNO_QSTR cxxflags ${MICROPY_CPP_FLAGS} -DNO_QSTR sources ${MICROPY_SOURCE_QSTR}) ++list(JOIN _qstr_args "\n" _qstr_args) ++if(CMAKE_VERSION VERSION_LESS 3.19) ++ file(GENERATE OUTPUT ${MICROPY_QSTR_ARGS_FILE} CONTENT "${_qstr_args}\n") ++else() ++ file(GENERATE OUTPUT ${MICROPY_QSTR_ARGS_FILE} CONTENT "${_qstr_args}\n" TARGET ${MICROPY_TARGET}) ++endif() ++ + # If any of the dependencies in this rule change then the C-preprocessor step must be run. + # It only needs to be passed the list of MICROPY_SOURCE_QSTR files that have changed since + # it was last run, but it looks like it's not possible to specify that with cmake. + add_custom_command( + OUTPUT ${MICROPY_QSTRDEFS_LAST} +- COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py pp ${CMAKE_C_COMPILER} -E output ${MICROPY_GENHDR_DIR}/qstr.i.last cflags ${MICROPY_CPP_FLAGS} -DNO_QSTR cxxflags ${MICROPY_CPP_FLAGS} -DNO_QSTR sources ${MICROPY_SOURCE_QSTR} ++ COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py pp ${CMAKE_C_COMPILER} -E output ${MICROPY_GENHDR_DIR}/qstr.i.last @${MICROPY_QSTR_ARGS_FILE} + WORKING_DIRECTORY ${MICROPY_CPP_WORKING_DIR} + DEPENDS ${MICROPY_MPVERSION} ++ ${MICROPY_QSTR_ARGS_FILE} + ${MICROPY_SOURCE_QSTR} + VERBATIM + COMMAND_EXPAND_LISTS +diff --git a/py/mkrules.mk b/py/mkrules.mk +index 19acea62e..e82304c2e 100644 +--- a/py/mkrules.mk ++++ b/py/mkrules.mk +@@ -139,13 +139,21 @@ $(BUILD)/%.o: $(BUILD)/%.c + # to get built before we try to compile any of them. + $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ_EXTRA_ORDER_DEPS) + +-# The file lists for makeqstrdefs.py pp. Windows caps a command line at 32767 +-# characters, which a build with large user C modules passes (every source is +-# named twice on a fresh build: $^ and $?), so there they go in a response file +-# that makeqstrdefs.py expands. $(file) needs GNU make 4.0; Windows' (MSYS2) has +-# it, and other hosts keep the command line as it was. +-ifeq ($(OS),Windows_NT) +-QSTR_PP_FILES = $(file >$(HEADER_BUILD)/qstr.i.args,sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $?)@$(HEADER_BUILD)/qstr.i.args ++# The file lists for makeqstrdefs.py pp. They go in a response file, one word ++# per line, that makeqstrdefs.py expands: make runs a command as one argument ++# to the shell, which Linux caps at 128 KiB and Windows at 32767 characters, and ++# a build with large user C modules names every source twice on a fresh build ++# ($^ and $?), so the command grew with each module and each character of the ++# tree's path. $(file) needs GNU make 4.0, so make 3.81 (macOS) keeps the ++# command line as it was. ++ifneq ($(filter-out 3.%,$(MAKE_VERSION)),) ++QSTR_PP_EMPTY := ++QSTR_PP_SPACE := $(QSTR_PP_EMPTY) $(QSTR_PP_EMPTY) ++define QSTR_PP_NEWLINE ++ ++ ++endef ++QSTR_PP_FILES = $(file >$(HEADER_BUILD)/qstr.i.args,$(subst $(QSTR_PP_SPACE),$(QSTR_PP_NEWLINE),$(strip sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $?)))@$(HEADER_BUILD)/qstr.i.args + else + QSTR_PP_FILES = sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $? + endif +-- +2.43.0 + diff --git a/provenance.json b/provenance.json index a4ca3282..19deecc1 100644 --- a/provenance.json +++ b/provenance.json @@ -85,6 +85,9 @@ }, "0029-micropython-py-mkrules-cmake-short-qstr-include-paths.patch": { "sha256": "1b675c07f807b56288d6626e1bfe52aeb21bb4bd567cd70c3d1cfc279c088b1f" + }, + "0030-micropython-py-mkrules-qstr-arguments-in-a-response-file.patch": { + "sha256": "2e4ef37460ef174c81c778c2274735d61894c4ad732efdd2574bea3ce6e6e337" } }, "tinyusb_patches": {