diff --git a/build_mp.py b/build_mp.py index 089b502e..2f4473a9 100755 --- a/build_mp.py +++ b/build_mp.py @@ -364,6 +364,12 @@ def esp32_board_dir(gen, base, board, variant, fragment_path, module_dirs): (gen / vfile).write_text("\n".join(vlines) + "\n") +def autosize_headroom(flash_bytes): + """Room left above an image that autosize grew to fit, so a few KB of growth + doesn't move the filesystem again: 1/32 of the flash, 64 KB to 256 KB.""" + return max(0x10000, min(0x40000, (flash_bytes // 32) & ~0xFFFF)) + + def esp32_autosize(log_text, build, port_dir, dest): """The table this build used, with its app partition grown to fit.""" m = re.search(r"app partition is too small for binary \S+ size (0x[0-9a-fA-F]+)", log_text) @@ -394,7 +400,8 @@ def num(v): mult = 1024 if v.endswith("k") else 1024 * 1024 if v.endswith("m") else 1 return int(v.rstrip("km"), 0) * mult - align, headroom = 0x10000, 0x40000 + flash = re.search(r'^CONFIG_ESPTOOLPY_FLASHSIZE="(\d+)MB"$', sdk, re.M) + align, headroom = 0x10000, autosize_headroom(int(flash.group(1)) * 1024 * 1024 if flash else 0x400000) image = int(m.group(1), 16) size = (image + align - 1) & ~(align - 1) size = (size + headroom + align - 1) & ~(align - 1) @@ -411,6 +418,26 @@ def num(v): return dest +def rp2_autosize(log_text): + """MICROPY_HW_FLASH_STORAGE_BYTES that leaves the firmware room to fit, from + the linker's memory table; None if the log isn't a FLASH overflow.""" + if not re.search(r"region `FLASH' overflowed", log_text): + return None + units = {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3} + regions = {} + for name, used, uu, size, su in re.findall( + r"^\s*(FLASH\w*):\s+(\d+(?:\.\d+)?) (B|KB|MB|GB)\s+(\d+(?:\.\d+)?) (B|KB|MB|GB)", log_text, re.M): + regions[name] = (int(float(used) * units[uu]), int(float(size) * units[su])) + if "FLASH" not in regions or "FLASH_FS" not in regions: + return None + used = regions["FLASH"][0] + total = regions["FLASH"][1] + regions["FLASH_FS"][1] # FLASH_ROMFS keeps its own size + align, headroom = 0x10000, autosize_headroom(total) + firmware = (used + headroom + align - 1) & ~(align - 1) + storage = total - firmware + return storage if storage >= align else None + + # ---- the build ------------------------------------------------------------- def main(): @@ -582,6 +609,30 @@ def shell(cmd, log=None): if not got or Path(got.group(1)) != table: die(f"the build used {got.group(1) if got else 'no'} partition table, not {table} (cmods#29)") say("partition layout in this image:\n" + "\n".join(" " + l for l in table.read_text().splitlines() if l and not l.startswith("#"))) + elif port == "rp2": + shell(make + ["submodules"]) + storage = None + for attempt in (1, 2): + extra = [f"MICROPY_HW_FLASH_STORAGE_BYTES={storage}"] if storage else [] + rc, out = shell(make + extra, log=True) + if rc == 0 or attempt == 2: + break + storage = rp2_autosize(out) + if not storage: + break + if args.no_autosize: + say(f"\nThe firmware doesn't fit. MICROPY_HW_FLASH_STORAGE_BYTES={storage} would fit it.") + say("(--no-autosize, so nothing was rebuilt.)") + break + say(f"\nautosize: the firmware didn't fit; building once more with a {storage // 1024} KB filesystem.") + say(" Shrinking the filesystem moves it: a board flashed with this image comes") + say(" up with an empty filesystem (cmods#30).\n") + # The port runs CMake only when the build has no Makefile yet. + (build / "Makefile").unlink(missing_ok=True) + # What the image was built with, grown now or on an earlier run (CMake caches it). + cache = build / "CMakeCache.txt" + got = re.search(r"^MICROPY_HW_FLASH_STORAGE_BYTES:\w+=(\d+)$", cache.read_text(), re.M) if cache.exists() else None + rp2_storage = int(got.group(1)) if rc == 0 and got else None else: shell(make + ["submodules"]) rc = shell(make) @@ -593,6 +644,8 @@ def shell(cmd, log=None): if rc != 0: die(f"the build failed (make exit {rc})") rec = dict(want, complete=True) + if port == "rp2" and rp2_storage: + rec["rp2_storage_bytes"] = rp2_storage rec["micropython"] = git_out(mp, "describe", "--always", "--abbrev=12") rec["module_revisions"] = record(module_dirs) rec_path.write_text(json.dumps(rec, indent=1) + "\n") diff --git a/docs/build-plan.md b/docs/build-plan.md index a05e69b7..e5806f29 100644 --- a/docs/build-plan.md +++ b/docs/build-plan.md @@ -1,6 +1,6 @@ # The build plan -**Status:** draft for Brad's review, 2026-10-05. Nothing here is built yet. +**Status:** built, 2026-10-05. Steps 1 to 5 are done and step 6 was dropped (see "Order of work"); this page now records how the build works and why. You build PyDevices firmware with one command: @@ -109,14 +109,19 @@ it as `BOARD_DIR=`. Upstream's tree is never edited, so the ports need no patch for this. If the app doesn't fit its partition, the build reads the size out of the -error, grows the app partition to the image rounded up to 64 KB plus 256 KB -of headroom, moves every partition after it, and builds once more against +error, grows the app partition to the image plus headroom (1/32 of the flash, +64 KB to 256 KB), rounded up to 64 KB, moves every partition after it, and builds once more against that table, which lives only in the build dir. It then prints the new layout and warns that the filesystem moved. This port puts the filesystem after the last partition, so a board flashed with the new layout comes up with an empty filesystem (cmods#30). Dev boards hold nothing we keep, so this is on by default; `--no-autosize` refuses instead and prints the table that would fit. +rp2 gets the same treatment. When the link overflows the `FLASH` region, the +build reads the linker's memory table, gives the firmware the image plus the +same headroom, and builds once more with the rest as the filesystem +(`MICROPY_HW_FLASH_STORAGE_BYTES`, recorded in the build record). + Output lands in `builds//[/]/`. The last level is always there, so one build never nests inside another: with no `--variant` it's the name upstream's `make` uses, `standard` on unix, windows and webassembly and @@ -196,7 +201,7 @@ that wants one (the wasm one) requires it by name. partition tables went; the S3 keeps a coredump partition through `variants/esp32/partitions.esp32s3.csv`, found by the same convention as the fragments. -6. **Optional: the VARIANT_DIR spike.** Teaching the esp32 and rp2 ports to +6. **Dropped (Brad, 2026-10-05): the VARIANT_DIR spike.** Teaching the esp32 and rp2 ports to take a variant from outside the board dir would make the generated dir tidier, not possible; it already works without it. Whether it becomes an upstream PR is your call. diff --git a/docs/newcomers.md b/docs/newcomers.md index 11a29323..e7efd04b 100644 --- a/docs/newcomers.md +++ b/docs/newcomers.md @@ -33,6 +33,8 @@ on top. Add `--modules` to any of these: | Waveshare ESP32-S3-Touch-LCD-4.3 | `--port esp32 --board ESP32_GENERIC_S3 --variant SPIRAM_OCT --flash 8MB` | | LilyGO T-Embed S3 | `--port esp32 --board ESP32_GENERIC_S3 --variant T_EMBED --flash 16MB` | | WT32-SC01 Plus | `--port esp32 --board ESP32_GENERIC_S3 --flash 16MB` | +| Raspberry Pi Pico (RP2040) | `--port rp2 --board RPI_PICO` | +| Raspberry Pi Pico 2 (RP2350) | `--port rp2 --board RPI_PICO2` | A new board usually needs no file at all: pick the upstream board and variant that match its chip and PSRAM, and give `--flash` its flash size. Make @@ -59,9 +61,12 @@ module is a directory with a `manifest.py`, or a C module with its glue, like ulab. A module never brings another; nothing checks dependencies, so name ulab when you name audiocomponents, or use `all`. -On esp32 the build may grow the app partition to fit the image. That moves -the filesystem, so a board flashed with that image comes up with an empty -one. `--no-autosize` refuses instead. +On esp32 and rp2 the build may grow the firmware's share of flash to fit the +image (on esp32 the app partition, on rp2 the region before the filesystem), +leaving 1/32 of the flash as headroom, 64 KB to 256 KB. That moves the +filesystem, so a board flashed with that image comes up with an empty one. +`--no-autosize` refuses instead. On a Pico, `all` is far too big either way; +name what your project uses. ## esp32 settings diff --git a/modules.lock b/modules.lock index adf50532..1d99b00c 100644 --- a/modules.lock +++ b/modules.lock @@ -4,11 +4,11 @@ # modules/castif, tracked in this repository. Commits are full SHAs: GitHub # serves a fetch by commit only for the whole 40 characters. audiocomponents https://github.com/PyDevices/audiocomponents.git 883d591784c4560bf6073a1d2d10096cc7f2122f -audiodsp https://github.com/PyDevices/audiodsp.git 79616bae165b63069ccecc6d335c00ca334b193c +audiodsp https://github.com/PyDevices/audiodsp.git 7da36f7bf016236c8df02505a653ea35dc62d600 audioif https://github.com/PyDevices/audioif.git b6b35a48a0cd23ddc549f8c0643d77ffe778da28 cameraif https://github.com/PyDevices/cameraif.git 77b20b128800d5e1e23aab4c131e58f46b12cad2 -displayif https://github.com/PyDevices/displayif.git 7b955e50216e954db12fef9e4c278f41cbedf73a -lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git 821fc459e6210cf0ee441fc9a65a13a223ab8d84 +displayif https://github.com/PyDevices/displayif.git cd6f82c38ece2266b241c240542902d9465b6559 +lvgl-micropython https://github.com/PyDevices/lvgl-micropython.git 8f7d89d0f8170ef526d5bd798c4bc0d05651309e palettes https://github.com/PyDevices/palettes.git 6f0870f0c8a0b48c56d2150b3f800bbbee7761e1 pdwidgets https://github.com/PyDevices/pdwidgets.git 306c1e20e0c3c63066dab6d10155fcf165a6ba13 pydevices https://github.com/PyDevices/pydevices.git c55aa1e7da1c7db8e50536bf939dd38f27f588f5 diff --git a/scripts/exercise_build.sh b/scripts/exercise_build.sh index a3df1037..2ba31129 100755 --- a/scripts/exercise_build.sh +++ b/scripts/exercise_build.sh @@ -8,7 +8,8 @@ # Clones micropython-pydevices into a fresh directory outside any PyDevices # workspace, so build_mp.py has to fetch MicroPython, every module and each # toolchain itself. It checks the failure paths, builds every desktop port and -# two esp32 boards with --modules all, and writes a report. What should match +# two esp32 boards with --modules all, two Picos with the module sets a +# display project uses, and writes a report. What should match # on any machine: module commits, the frozen and built-in module lists, and an # esp32 build's sdkconfig and partition layout. What won't: sizes and SHAs, # which carry the build date and absolute paths. It needs no board, and a few @@ -16,7 +17,8 @@ # # Environment: # WORK where to clone and build (default ./exercise); must not exist -# TARGETS space-separated subset of: unix wasm windows s3 p4 (default all) +# TARGETS space-separated subset of: unix wasm windows s3 p4 pico pico2 +# (default all) # DEPS_FROM a directory holding esp-idf/, emsdk/ and SDL2/ to link instead # of fetching # @@ -49,16 +51,18 @@ for name, m in sorted(d.get("module_revisions", {}).items()): EOF rb_fz=$(find "$rb_dir" -name frozen_content.c -not -path "*/managed_components/*" | head -1) if [ -n "$rb_fz" ]; then - awk '/const char mp_frozen_names\[\]/,/^};/' "$rb_fz" | grep -o '"[^"]*\\0"' | sort > "$rb_dir/frozen-names.txt" + awk '/const char mp_frozen_names\[\]/,/^};/' "$rb_fz" | grep -o '"[^"]*\\0"' | LC_ALL=C sort > "$rb_dir/frozen-names.txt" say "frozen $(wc -l < "$rb_dir/frozen-names.txt") files, list sha $(sha "$rb_dir/frozen-names.txt")" fi rb_md=$(find "$rb_dir" -name moduledefs.h -path "*genhdr*" | head -1) if [ -n "$rb_md" ]; then - grep -o 'MODULE_DEF_[A-Za-z0-9_]*' "$rb_md" | sort -u > "$rb_dir/builtin-modules.txt" + grep -o 'MODULE_DEF_[A-Za-z0-9_]*' "$rb_md" | LC_ALL=C sort -u > "$rb_dir/builtin-modules.txt" say "builtin $(wc -l < "$rb_dir/builtin-modules.txt") modules, list sha $(sha "$rb_dir/builtin-modules.txt")" fi if [ -f "$rb_dir/sdkconfig" ]; then - grep '^CONFIG_' "$rb_dir/sdkconfig" | sort > "$rb_dir/sdkconfig.sorted" + # The partition table's path names the build dir: hash it as . + rb_abs=$(cd "$rb_dir" && pwd) + grep '^CONFIG_' "$rb_dir/sdkconfig" | sed "s|$rb_abs||g" | LC_ALL=C sort > "$rb_dir/sdkconfig.sorted" say "sdkconfig $(wc -l < "$rb_dir/sdkconfig.sorted") settings, sha $(sha "$rb_dir/sdkconfig.sorted")" for rb_k in ESPTOOLPY_FLASHSIZE FREERTOS_HZ SPI_FLASH_AUTO_SUSPEND ESP_COREDUMP_ENABLE_TO_FLASH \ USB_HOST_HUBS_SUPPORTED USB_HOST_HW_BUFFER_BIAS_IN CAMERA_OV5647 BT_NIMBLE_HS_FLOW_CTRL \ @@ -70,7 +74,7 @@ EOF say "partitions (autosize grew the app):" grep -v '^#' "$rb_dir/partitions.csv" | grep . | sed 's/^/ /' | tee -a "$REPORT" fi - for rb_f in micropython micropython.exe micropython.mjs micropython.wasm micropython.bin firmware.bin; do + for rb_f in micropython micropython.exe micropython.mjs micropython.wasm micropython.bin firmware.bin firmware.uf2; do [ -f "$rb_dir/$rb_f" ] && say "output $rb_f $(stat -c %s "$rb_dir/$rb_f") bytes, sha $(sha "$rb_dir/$rb_f")" done return 0 @@ -121,7 +125,7 @@ main() { REF=${1:-main} WORK=${WORK:-$PWD/exercise} - TARGETS=${TARGETS:-unix wasm windows s3 p4} + TARGETS=${TARGETS:-unix wasm windows s3 p4 pico pico2} # The desktop board config opens an SDL window; a headless machine has no display. SDL_VIDEODRIVER=${SDL_VIDEODRIVER:-dummy} export SDL_VIDEODRIVER @@ -144,6 +148,7 @@ main() { say "python3 $(python3 --version 2>&1)" say "mingw $(x86_64-w64-mingw32-gcc --version 2>/dev/null | head -1 || echo missing)" say "node $(node --version 2>/dev/null || echo missing)" + say "arm $(arm-none-eabi-gcc --version 2>/dev/null | head -1 || echo missing)" section "clone" git clone -q --branch "$REF" "$URL" "$WORK/micropython-pydevices" < /dev/null || { say "clone failed"; return 1; } @@ -208,6 +213,18 @@ print('RUN', sys.implementation._machine, np.sum(np.array([1, 2, 3])), board_con build p4-devkit --port esp32 --board ESP32_GENERIC_P4 --variant C6_WIFI --flash 16MB --modules all fi + # rp2: what a display project on a Pico carries, not all (which doesn't fit). + for m_t in pico pico2; do + want $m_t || continue + if ! command -v arm-none-eabi-gcc > /dev/null; then + section "build $m_t: SKIPPED, arm-none-eabi-gcc is missing (apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi)" + elif [ $m_t = pico ]; then + build pico --port rp2 --board RPI_PICO --modules displayif,pygraphics,palettes,pdwidgets + else + build pico2 --port rp2 --board RPI_PICO2 --modules displayif,lvgl-micropython + fi + done + section "done" say "report: $REPORT" say "logs: $LOGS"