Python support: transpiled Python on every target, Python servers, Pillow and font-from-data

.github/workflows/developer-guide-docs.yml

@@ -179,11 +179,12 @@ jobs:
           # Through retry.sh, transient failures only, like the demo install below:
           # this is the step that resolves the most from Central, and a single 502
           # on a plugin dependency (maven-shared-utils) failed the whole guide build.
-          # dart-runtime and flutter-runtime: docs/demos/common depends on the Flutter
-          # runtime at this version, which nothing else selected here pulls in; they are
+          # dart-runtime, flutter-runtime and python-runtime: docs/demos/common depends on
+          # the Flutter and Python runtimes at this version, which nothing else selected
+          # here pulls in; they are
           # in the reactor because this job runs on JDK 17 (the flutter-modules profile).
           RETRY_ONLY_MATCHING=transient xvfb-run -a bash scripts/ci/retry.sh mvn -B -ntp -f maven/pom.xml \
-            -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime \
+            -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend,backend-test,dart-runtime,flutter-runtime,python-runtime \
             -am install \
             -Plocal-dev-javase \
             -DskipTests \

.github/workflows/parparvm-tests.yml

@@ -34,6 +34,16 @@ on:
       - 'maven/project-model/**'
       - 'maven/backend/**'
       - 'maven/backend-test/**'
+      # The Python SQLAlchemy step runs these against the PostgreSQL service.
+      - 'maven/python-runtime/**'
+      - 'maven/python-transpiler/**'
+      - 'scripts/python-backend/sqlalchemy-realdb-test.sh'
+      - 'scripts/python-backend/build.sh'
+      - 'scripts/python-backend/e2e-test.sh'
+      - 'scripts/python-backend/e2e_client.py'
+      - 'scripts/python-backend/samples/**'
+      # sqlalchemy-realdb-test.sh compiles the host file system shims from here.
+      - 'scripts/python-bench/hostfs/**'
       - 'scripts/hellocodenameone/pom.xml'
       - 'scripts/hellocodenameone/backend/**'
       # The core classes the backend compiles as its own (the ORM and the entity
@@ -120,6 +130,16 @@ on:
       - 'maven/project-model/**'
       - 'maven/backend/**'
       - 'maven/backend-test/**'
+      # The Python SQLAlchemy step runs these against the PostgreSQL service.
+      - 'maven/python-runtime/**'
+      - 'maven/python-transpiler/**'
+      - 'scripts/python-backend/sqlalchemy-realdb-test.sh'
+      - 'scripts/python-backend/build.sh'
+      - 'scripts/python-backend/e2e-test.sh'
+      - 'scripts/python-backend/e2e_client.py'
+      - 'scripts/python-backend/samples/**'
+      # sqlalchemy-realdb-test.sh compiles the host file system shims from here.
+      - 'scripts/python-bench/hostfs/**'
       - 'scripts/hellocodenameone/pom.xml'
       - 'scripts/hellocodenameone/backend/**'
       # The core classes the backend compiles as its own (the ORM and the entity
@@ -492,6 +512,45 @@ jobs:
           # the step above just installed from this same checkout.
           CN1SS_BACKEND_SKIP_INSTALL: 1
 
+      # The SQLAlchemy conformance programs against the PostgreSQL service
+      # above: the suite in python-transpiler runs them on SQLite only, so this
+      # is the one place the postgresql dialect and the _cn1db bridge meet a
+      # real server. PostgreSQL and the job's MySQL service; MariaDB is run by
+      # hand. Last of the steps using MySQL: each program drops and recreates
+      # the cn1 database.
+      # python-runtime compiles at release 17, hence JDK 17 for the install.
+      - name: Python SQLAlchemy on PostgreSQL and MySQL
+        run: |
+          (cd maven && bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -pl python-runtime,python-transpiler \
+            install -DskipTests -Dspotbugs.skip=true -Dpmd.skip=true -Dcheckstyle.skip=true \
+            -Dmaven.javadoc.skip=true -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo")
+          # The test transpiles against JavaAPI from the same repository, and
+          # the step that installs it for the native samples runs after this.
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \
+            -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo"
+          bash scripts/python-backend/sqlalchemy-realdb-test.sh "$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_17_HOME }}
+          JAVA17_HOME: ${{ env.JDK_17_HOME }}
+          CN1_SQLA_POSTGRES: postgres://cn1:cn1secret@127.0.0.1:5432/cn1?sslmode=disable
+          CN1_SQLA_MYSQL: mysql://cn1:cn1secret@127.0.0.1:3306/cn1?sslmode=disable
+
+      # The Flask and FastAPI samples built NATIVELY (ParparVM, clang) and
+      # driven over sockets by the same client as the JVM run. The only place
+      # Python handlers run on the backend's virtual threads, so the only run
+      # of the GIL's scheduler path. python-transpiler and python-runtime were
+      # installed by the step above; core, backend and build-engine by the
+      # contract install; JavaAPI explicitly, since the JVM tests may not.
+      - name: Python backend end to end, native
+        run: |
+          bash "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -B -f vm/pom.xml -pl JavaAPI install -DskipTests \
+            -Dmaven.repo.local="$GITHUB_WORKSPACE/.m2-repo"
+          bash scripts/python-backend/e2e-test.sh --native "$GITHUB_WORKSPACE/.m2-repo"
+        env:
+          JAVA_HOME: ${{ env.JDK_8_HOME }}
+          JDK_8_HOME: ${{ env.JDK_8_HOME }}
+          JAVA17_HOME: ${{ env.JDK_17_HOME }}
+
       - name: Run ParparVM Benchmark
         working-directory: vm
         # Each benchmark already creates its own mutator/collector threads. Run

.github/workflows/pr.yml

@@ -402,7 +402,15 @@ jobs:
             -Dcn1.binaries="${CN1_BINARIES}" \
             -Pcompile-android \
             -DbackendAnalysis \
-            -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,compose-compat,fxml-compiler,desktop-compat-runtime -am verify
+            -pl android,ios,project-model,build-engine,codenameone-maven-plugin,codenameone-gradle-plugin,backend,backend-test,backend-parparvm,javac,android-res-compiler,android-compat,swing-compat,javafx-compat,compose-compat,fxml-compiler,desktop-compat-runtime,python-transpiler -am verify
+      # The Python runtime compiles at release 17, so it cannot join the JDK 8
+      # reactor above; it runs against the core that reactor installed. It ships
+      # inside applications, so it is held to the same zero-findings gate.
+      - name: Run SpotBugs for the Python runtime
+        if: ${{ matrix.java-version == 8 }}
+        working-directory: maven
+        run: |
+          JAVA_HOME="${JAVA_HOME_17}" mvn -B -DskipTests=true -Pflutter-modules -pl python-runtime verify
       - name: Type-check the SQLCipher-backed database package
         if: ${{ matrix.java-version == 8 }}
         env:
@@ -620,7 +628,7 @@ jobs:
       - name: Generate static analysis HTML summaries
         if: ${{ !cancelled() && matrix.java-version == 8 }}
         env:
-          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target
+          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target
           QUALITY_REPORT_SERVER_URL: ${{ github.server_url }}
           QUALITY_REPORT_REPOSITORY: ${{ github.repository }}
           QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }}
@@ -777,16 +785,16 @@ jobs:
       - name: Generate quality report summary
         if: ${{ !cancelled() && matrix.java-version == 8 }}
         env:
-          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target
+          QUALITY_REPORT_TARGET_DIRS: maven/core-unittests/target:maven/android/target:maven/ios/target:vm/ByteCodeTranslator/target:maven/codenameone-maven-plugin/target:maven/build-engine/target:maven/project-model/target:maven/codenameone-gradle-plugin/target:maven/build-hint-catalog/target:maven/build-hint-tools/target:maven/backend/target:maven/backend-test/target:maven/backend-parparvm/target:maven/javac/target:maven/android-res-compiler/target:maven/android-compat/target:maven/swing-compat/target:maven/javafx-compat/target:maven/compose-compat/target:maven/fxml-compiler/target:maven/desktop-compat-runtime/target:maven/python-transpiler/target:maven/python-runtime/target
           # Every project listed here must produce a SpotBugs report and that
           # report must be empty. A missing report fails the gate too, so a
           # SpotBugs run that silently stops running is not mistaken for clean.
-          QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:compose-compat:fxml-compiler:desktop-compat-runtime
-          # PMD and Checkstyle run on the core and on the backend runtime, and each
-          # must produce its report: without one, the PMD gate below it
-          # has nothing to read and passes.
-          QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm
-          QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test
+          QUALITY_REPORT_REQUIRED_SPOTBUGS: core-unittests:android:ios:ByteCodeTranslator:codenameone-maven-plugin:build-engine:project-model:codenameone-gradle-plugin:build-hint-catalog:build-hint-tools:backend:backend-test:javac:android-res-compiler:android-compat:swing-compat:javafx-compat:compose-compat:fxml-compiler:desktop-compat-runtime:python-transpiler:python-runtime
+          # PMD and Checkstyle run on the core, the backend runtime and the Python
+          # transpiler and runtime, and each must produce its report: without one,
+          # the PMD gate below it has nothing to read and passes.
+          QUALITY_REPORT_REQUIRED_PMD: core-unittests:backend:backend-test:backend-parparvm:python-transpiler:python-runtime
+          QUALITY_REPORT_REQUIRED_CHECKSTYLE: core-unittests:backend:backend-test:python-transpiler:python-runtime
           QUALITY_REPORT_SERVER_URL: ${{ github.server_url }}
           QUALITY_REPORT_REPOSITORY: ${{ github.repository }}
           QUALITY_REPORT_REF: ${{ github.event.pull_request.head.sha || github.sha }}

.github/workflows/python-bench.yml

@@ -0,0 +1,204 @@
+name: Python benchmark
+
+# Runs the kernels in scripts/python-bench/kernels twice -- by CPython and
+# transpiled to Java by Codename One -- checks that both compute the same
+# result, and gates the ours/CPython ratio of compute time, process wall time
+# and peak memory against scripts/python-bench/baseline/ratios.json.
+#
+# Two arms: the JVM (the simulator and desktop targets) and ParparVM (the
+# native translation iOS and the native desktop targets ship), each gated on
+# its own baseline. The ParparVM arm also runs the conformance programs as
+# native executables, where a bad cast is a crash rather than an exception.
+# Android and JavaScript join as the Python support grows.
+#
+# It also runs the transpiler's conformance tests, which need the Java 17
+# runtime and so cannot run in pr.yml's JDK 8 reactor.
+
+on:
+  pull_request:
+    paths:
+      - 'maven/python-transpiler/**'
+      - 'maven/python-runtime/**'
+      - 'scripts/python-bench/**'
+      - '.github/workflows/python-bench.yml'
+      # The ParparVM arm translates and compiles with these.
+      - 'vm/ByteCodeTranslator/**'
+      - 'vm/JavaAPI/**'
+      - 'vm/selfhost/compile-dist.sh'
+      # Transpiled Python runs on the core's classes (collections, io/gzip).
+      - 'CodenameOne/src/**'
+      - 'maven/core/**'
+      - 'maven/pom.xml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  # The pr-ci-container image lives on ghcr; without this the job dies in
+  # "Initialize containers" with a bare "denied".
+  packages: read
+
+env:
+  # Pinned, like flutter-bench's FLUTTER_REF: the ratios are against this
+  # interpreter, and moving it is a deliberate re-baseline.
+  CPYTHON_VERSION: '3.13.9'
+
+jobs:
+  # JDK 8 for the core (the container bakes both JDKs and cn1-binaries, as
+  # flutter-bench's framework job explains), JDK 17 for the runtime.
+  framework:
+    name: framework
+    runs-on: ubuntu-latest
+    container: ghcr.io/codenameone/codenameone/pr-ci-container:latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Link cn1-binaries
+        run: ln -s /opt/cn1-binaries ../cn1-binaries
+
+      # compat-testing and backend are the transpiler's test dependencies
+      # (the pilclient_ and pil_ conformance programs); the JDK 17 step
+      # below runs without -am, so they have to be installed here.
+      - name: Build the core and the transpiler on JDK 8
+        run: |
+          cd maven
+          JAVA_HOME="$JAVA_HOME_8" bash ../scripts/ci/retry.sh mvn -B -q install -DskipTests \
+              -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \
+              -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \
+              -pl core,python-transpiler,compat-testing,backend -am
+
+      # Without -am: core was installed above, and re-resolving it here would
+      # rebuild it on the wrong JDK.
+      - name: Build the runtime and run the conformance tests on JDK 17
+        run: |
+          cd maven
+          JAVA_HOME="$JAVA_HOME_17" bash ../scripts/ci/retry.sh mvn -B install \
+              -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \
+              -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \
+              -pl python-runtime,python-transpiler
+
+      - name: Build the ParparVM translator and its class library on JDK 8
+        run: |
+          JAVA_HOME="$JAVA_HOME_8" bash scripts/ci/retry.sh mvn -B -q install -DskipTests \
+              -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \
+              -Dmaven.repo.local="$GITHUB_WORKSPACE/m2" \
+              -f vm/pom.xml -pl JavaAPI,ByteCodeTranslator -am
+
+      - name: Pack the repository
+        run: tar -C "$GITHUB_WORKSPACE" -czf framework-m2.tgz m2
+
+      - name: Upload the repository
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-bench-framework
+          path: framework-m2.tgz
+          retention-days: 1
+
+  measure:
+    name: jvm
+    needs: framework
+    # Pinned: setup-python has no CPYTHON_VERSION build for ubuntu-latest
+    # (26.04), and the baselines are measured against that exact version.
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Set up CPython
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.CPYTHON_VERSION }}
+
+      - name: Download the framework
+        uses: actions/download-artifact@v4
+        with:
+          name: python-bench-framework
+
+      - name: Unpack the framework
+        run: tar -xzf framework-m2.tgz
+
+      - name: Measure and gate
+        run: |
+          repo="$GITHUB_WORKSPACE/m2/com/codenameone"
+          v=$(cd "$repo/codenameone-python-runtime" && ls -d */ | head -1 | tr -d /)
+          python3 scripts/python-bench/run_bench.py \
+              --java-home "$JAVA_HOME" \
+              --runtime-jar "$repo/codenameone-python-runtime/$v/codenameone-python-runtime-$v.jar" \
+              --core-jar "$repo/codenameone-core/$v/codenameone-core-$v.jar" \
+              --transpiler-cp "$repo/codenameone-python-transpiler/$v/codenameone-python-transpiler-$v.jar" \
+              --json python-bench.json --check | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload the report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-bench-report
+          path: python-bench.json
+
+  parparvm:
+    name: parparvm
+    needs: framework
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      PYBENCH_M2: ${{ github.workspace }}/m2
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install clang
+        run: |
+          bash scripts/ci/apt-get-update.sh
+          sudo apt-get install -y clang time
+
+      # The translator runs on JDK 8, the generated Java compiles on JDK 17.
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: zulu
+          java-version: '8'
+
+      - name: Remember JDK 8
+        run: echo "JDK_8_HOME=$JAVA_HOME" >> "$GITHUB_ENV"
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+
+      - name: Remember JDK 17
+        run: echo "JAVA17_HOME=$JAVA_HOME" >> "$GITHUB_ENV"
+
+      - name: Set up CPython
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.CPYTHON_VERSION }}
+
+      - name: Download the framework
+        uses: actions/download-artifact@v4
+        with:
+          name: python-bench-framework
+
+      - name: Unpack the framework
+        run: tar -xzf framework-m2.tgz
+
+      - name: Conformance, natively
+        run: scripts/python-bench/conformance_parparvm.sh
+
+      - name: Measure and gate
+        run: |
+          python3 scripts/python-bench/parparvm_table.py --json python-bench-parparvm.json --check \
+              | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload the report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-bench-parparvm-report
+          path: python-bench-parparvm.json