Build: drop cached packages and the dist that no longer match the recipe pins

build_android.sh

@@ -396,8 +396,16 @@ else
 fi
 
 cd "$APP_DIR"
+# p4a keeps older installed package versions and reuses an existing dist whose
+# bundle was built from them; drop whatever no longer matches the recipe pins
+# (not the compiled python3/sdl2 caches) so the APK carries only this build.
+if [[ -d "$APP_DIR/.buildozer/android/platform" ]]; then
+    "$PYTHON" "$SCRIPT_DIR/scripts/stale_dist.py" prune "$APP_DIR/.buildozer/android/platform"
+fi
+BUILD_START_MARK=$(mktemp "${TMPDIR:-/tmp}/android-template-build.XXXXXX")
+trap 'rm -f "$BUILD_START_MARK"' EXIT
 # Always "android debug" — never "android clean". Extra args are flags only.
 "$BUILDOZER" android debug "${BUILDOZER_ARGS[@]}"
 
-echo "==> APK output:"
-ls -1 "$APP_DIR"/bin/*.apk
+echo "==> APK output (this build only; older APKs stay in $APP_DIR/bin):"
+find "$APP_DIR/bin" -name '*.apk' -newer "$BUILD_START_MARK" | sort

docs/building.md

@@ -34,14 +34,9 @@ sed -i "s#<old checkout>#<new checkout>#g" p4a_app/.buildozer/android/platform/b
 
 ### After you change a recipe's pin
 
-An incremental build doesn't pick up a new `version` in `p4a_recipes/`. p4a matches the existing dist by its recipe names alone, so it repackages the packages it installed last time. Remove the dist and the old version's dist-info (here, moving pydevices-lvgl off 9.5.49), then build again:
+Just build again. p4a matches an existing dist by its recipe names alone and keeps older installed versions beside newer ones, so on its own it would repackage whatever it installed last time. `build_android.sh` drops any cached package (and the dist built from it) whose version isn't the pin in `p4a_recipes/`, and the p4a hook (`scripts/p4a_hook.py`, wired in by `p4a.hook` in `buildozer.spec`) fails the build if the dist still differs from the pins. Every Python recipe that moved reinstalls; the compiled ones (CPython, SDL2) are reused. `scripts/stale_dist.py` holds the logic if you want to run `prune` or `check` by hand.
 
-```bash
-B=p4a_app/.buildozer/android/platform/build-arm64-v8a_x86_64
-rm -rf $B/dists/<dist> $B/build/python-installs/<dist>/*/pydevices_lvgl-9.5.49.dist-info
-```
-
-Every Python recipe reinstalls; the compiled ones (CPython, SDL2) are reused. To see what an APK carries, list the dist-info folders in its bundle:
+To see what an APK carries, list the dist-info folders in its bundle:
 
 ```bash
 unzip -p p4a_app/bin/<apk> lib/arm64-v8a/libpybundle.so | tar -tz | grep 'dist-info/$'

p4a_app/buildozer.spec

@@ -46,6 +46,9 @@ p4a.extra_args = --extra-index-url https://test.pypi.org/simple/ --extra-index-u
 
 # Thin PyProjectRecipe wrappers that install matching TestPyPI / PyPI wheels.
 p4a.local_recipes = ../p4a_recipes
+# Refuses a dist whose Python packages differ from the recipe pins
+# (see scripts/p4a_hook.py).
+p4a.hook = ../scripts/p4a_hook.py
 
 [buildozer]
 log_level = 2

scripts/p4a_hook.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: MIT
+"""python-for-android hooks (wired in via ``p4a.hook`` in buildozer.spec).
+
+``before_apk_build`` runs inside the dist directory that gradle is about to
+compile. It refuses a dist whose Python packages differ from the recipe pins
+(``stale_dist.py``): p4a reuses an existing dist and keeps older installed
+versions beside newer ones, so a moved pin could ship the old build.
+"""
+
+import os
+import pathlib
+import sys
+
+_HERE = pathlib.Path(__file__).resolve().parent
+if str(_HERE) not in sys.path:
+    sys.path.insert(0, str(_HERE))
+
+import stale_dist  # noqa: E402
+
+
+def before_apk_build(toolchain):
+    sites = stale_dist.check(os.getcwd())
+    print("p4a_hook: dist packages match the recipe pins (%d bundle(s))" % sites)

scripts/stale_dist.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: MIT
+"""Keep a build from packaging an older build's Python packages.
+
+python-for-android reuses two things between builds: the per-arch install
+directory (``build/python-installs/<dist>/<arch>``) and, when the recipe list is
+unchanged, the whole dist (``dists/<dist>``, which holds the Python bundle that
+ends up in the APK). Neither is refreshed when a recipe pin moves. Install a
+newer ``pydevices`` over an older one and both ``*.dist-info`` directories
+stay; move a pin back, or to any version that is already installed, and the
+new pin is ignored. The APK then ships whatever was installed first.
+
+``prune`` runs before the build and removes any package whose installed
+versions are not exactly the recipe pin (the dist too, so its bundle is built
+again). ``check`` runs from the p4a hook once the dist exists and fails the
+build if a package is still wrong. Neither touches hostpython, python3, sdl2 or
+any other cached compile.
+
+    python3 scripts/stale_dist.py prune <platform dir> [recipes dir]
+    python3 scripts/stale_dist.py check <dist dir> [recipes dir]
+"""
+
+import pathlib
+import re
+import shutil
+import sys
+
+_HERE = pathlib.Path(__file__).resolve().parent
+RECIPES = _HERE.parent / "p4a_recipes"
+
+# Recipes whose pip distribution is not named like the recipe.
+_PIP_NAMES = {"pydeviceslvgl": "pydevices-lvgl"}
+
+
+def norm(name):
+    return re.sub(r"[-_.]+", "_", name).lower()
+
+
+def pins(recipes=RECIPES):
+    """{normalized pip name: pinned version} for every pinned recipe."""
+    found = {}
+    for init in sorted(pathlib.Path(recipes).glob("*/__init__.py")):
+        text = init.read_text()
+        name = re.search(r'^\s+name\s*=\s*"([^"]+)"', text, re.M)
+        version = re.search(r'^\s+version\s*=\s*"([^"]+)"', text, re.M)
+        if name and version:
+            found[norm(_PIP_NAMES.get(name.group(1), name.group(1)))] = version.group(1)
+    return found
+
+
+def installed(site):
+    """{normalized name: [versions]} from the *.dist-info directories in site."""
+    found = {}
+    for info in pathlib.Path(site).glob("*.dist-info"):
+        name, _, version = info.name[: -len(".dist-info")].rpartition("-")
+        found.setdefault(norm(name), []).append(version)
+    return found
+
+
+def problems(site, pinned):
+    """Packages in site that are not exactly the pinned version."""
+    out = []
+    for name, versions in sorted(installed(site).items()):
+        if name in pinned and versions != [pinned[name]]:
+            out.append("%s: installed %s, pinned %s" % (name, ", ".join(sorted(versions)), pinned[name]))
+    return out
+
+
+def uninstall(site, name):
+    """Remove every version of a package from site, using its RECORD."""
+    site = pathlib.Path(site)
+    for info in list(site.glob("*.dist-info")):
+        if norm(info.name[: -len(".dist-info")].rpartition("-")[0]) != name:
+            continue
+        record = info / "RECORD"
+        if record.is_file():
+            for line in record.read_text().splitlines():
+                rel = line.rsplit(",", 2)[0]
+                target = site / rel
+                if rel and (target.is_file() or target.is_symlink()):
+                    target.unlink()
+        shutil.rmtree(info)
+    for directory in sorted((p for p in site.rglob("*") if p.is_dir()), key=lambda p: -len(p.parts)):
+        if not any(directory.iterdir()):
+            directory.rmdir()
+
+
+def bundle_sites(dist):
+    return sorted(pathlib.Path(dist).glob("_python_bundle__*/_python_bundle/site-packages"))
+
+
+def prune(platform, recipes=RECIPES):
+    pinned = pins(recipes)
+    any_change = False
+    for build in sorted(pathlib.Path(platform).glob("build-*")):
+        changed = False
+        for site in sorted(build.glob("build/python-installs/*/*")):
+            for line in problems(site, pinned):
+                name = line.split(":")[0]
+                print("stale_dist: %s (%s)" % (line, site))
+                uninstall(site, name)
+                changed = True
+                any_change = True
+        for dist in sorted(build.glob("dists/*")):
+            bad = [p for site in bundle_sites(dist) for p in problems(site, pinned)]
+            if bad or changed:
+                print("stale_dist: dropping dist %s so its bundle is built again" % dist)
+                shutil.rmtree(dist)
+    if not any_change:
+        print("stale_dist: installed packages match the recipe pins")
+
+
+def check(dist, recipes=RECIPES):
+    pinned = pins(recipes)
+    bad = [p for site in bundle_sites(dist) for p in problems(site, pinned)]
+    if bad:
+        raise RuntimeError(
+            "dist %s carries packages that differ from the recipe pins:\n  %s" % (dist, "\n  ".join(bad))
+        )
+    return len(bundle_sites(dist))
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3 or sys.argv[1] not in ("prune", "check"):
+        sys.exit(__doc__)
+    recipes = sys.argv[3] if len(sys.argv) > 3 else RECIPES
+    if sys.argv[1] == "prune":
+        prune(sys.argv[2], recipes)
+    else:
+        check(sys.argv[2], recipes)
+        print("stale_dist: dist packages match the recipe pins")